@popmelt.com/core 0.16.0 → 0.18.0

package/README.md

@@ -243,10 +243,10 @@ If the toolbar opens but cannot reach an AI, check that:
 - The framework plugin or config wrapper is installed
 - The development server was restarted after installation
 
-## Image annotation
+## Media annotation
 
-Manual image annotation is enabled by default in `PopmeltProvider`, without a
-vision dependency or model download.
+Manual image and video annotation are enabled by default in `PopmeltProvider`,
+without a vision dependency or model download.
 
 ```tsx
 <PopmeltProvider>{children}</PopmeltProvider>
@@ -254,23 +254,31 @@ vision dependency or model download.
 // Optional: disable image annotation.
 <PopmeltProvider imageAnnotation={false}>{children}</PopmeltProvider>
 
+// Optional: disable video annotation.
+<PopmeltProvider videoAnnotation={false}>{children}</PopmeltProvider>
+
 // Optional: limit image discovery (all img elements are included by default).
 <PopmeltProvider imageAnnotation={{ selector: 'img[data-popmelt-media]' }}>
   {children}
 </PopmeltProvider>
 ```
 
-In Chat, drag over an image to create a named object. Double-click an object to
+In Comment, drag over an image to create a named object. Double-click an object to
 rename or resize it, single-click it to comment, and right-click its label to
 delete it. Saved targets stay aligned as the image resizes.
 
+The same interactions work on a paused video. A manual box stays at the saved
+position unless a perception extension supplies motion tracking. Motion tracks
+produced by an extension are persisted and remain replayable by Core alone.
+
 Image targets use browser-local storage by default. Supply
 `imageAnnotation.store` when the project needs another persistence layer. Add a
 stable `data-popmelt-media-id` to images whose URL may change or whose source
-appears more than once:
+appears more than once. Use the same attribute for durable video identities:
 
 ```tsx
 <img data-popmelt-media-id="hero-product" src={imageUrl} alt="Product" />
+<video data-popmelt-media-id="hero-reel" src={videoUrl} controls />
 ```
 
 No recognition or model download occurs unless See is installed and enabled.
@@ -278,8 +286,8 @@ No recognition or model download occurs unless See is installed and enabled.
 ## Add local object recognition with See
 
 [`@popmelt.com/see`](https://www.npmjs.com/package/@popmelt.com/see) adds
-browser-local object detection to the existing image-annotation experience. It
-requires Core 0.16.0 or newer.
+browser-local object detection and optional video tracking to the existing media
+annotation experience. It requires Core 0.17.0 or newer.
 
 Install it:
 
@@ -297,12 +305,12 @@ import { see } from '@popmelt.com/see';
 ```
 
 Recognition loads only when it is needed. Other extensions can be included in
-the same array. Model selection, self-hosting, and recognition settings are
+the same array. Model selection, self-hosting, recognition, and video tracking are
 documented in the See package README.
 
 ## Data and privacy
 
-Popmelt stores its annotations, screenshots, threads, UI preferences, and Imprint locally under `.popmelt/`. Unsent annotation drafts, preview state, and the default image-object store use origin-scoped browser storage. Supply a project-backed `imageAnnotation.store` when image targets must survive browser-site-data removal or move between origins.
+Popmelt stores its annotations, screenshots, threads, UI preferences, and Imprint locally under `.popmelt/`. Unsent annotation drafts, preview state, and the default media-object stores use origin-scoped browser storage. Supply project-backed `imageAnnotation.store` and `videoAnnotation.store` implementations when media targets must survive browser-site-data removal or move between origins.
 
 When you send a thread, the context needed to complete that work is passed to your selected AI provider and is subject to that provider's privacy policy.
 

package/codex-plugin/.codex-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "popmelt-core",
-  "version": "0.16.0",
+  "version": "0.18.0",
   "description": "Ambient project memory and shared understanding for Codex",
   "author": {
     "name": "Popmelt",

package/dist/adapter-three.d.mts

@@ -1,5 +1,19 @@
 import { Camera, Object3D, Raycaster, WebGLRenderer } from 'three';
 
+type AnnotationRevealStep = {
+    triggerSelector: string;
+    targetSelector?: string;
+    label: string;
+};
+type AnnotationRevealRecipe = {
+    steps: AnnotationRevealStep[];
+    location: string[];
+};
+type ReactSourceLocation = {
+    fileName: string;
+    lineNumber: number;
+    columnNumber?: number;
+};
 /** Coordinates refer to the original image, not its responsive DOM rectangle. */
 type ImageObjectContext = {
     src: string;
@@ -19,6 +33,47 @@ type ImageObjectContext = {
         height: number;
     };
 };
+/** Coordinates refer to the decoded video frame, not its responsive DOM box. */
+type VideoObjectContext = {
+    src: string;
+    width: number;
+    height: number;
+    duration: number;
+    currentTime: number;
+    normalizedBox: {
+        x: number;
+        y: number;
+        width: number;
+        height: number;
+    };
+    box: {
+        x: number;
+        y: number;
+        width: number;
+        height: number;
+    };
+    trackStart: number;
+    trackEnd: number;
+};
+type ElementInfo = {
+    selector: string;
+    tagName: string;
+    /** Adapter-owned semantic name for a non-DOM subject. */
+    subjectLabel?: string;
+    id?: string;
+    className?: string;
+    textContent?: string;
+    dataAttributes?: Record<string, string>;
+    reactComponent?: string;
+    reactComponentPath?: string[];
+    reactSource?: ReactSourceLocation;
+    imageObject?: ImageObjectContext;
+    videoObject?: VideoObjectContext;
+    /** The real DOM surface, rather than the subject's hidden projection. */
+    sourceElement?: ElementInfo;
+    context?: string;
+    revealRecipe?: AnnotationRevealRecipe;
+};
 
 type PopmeltSubjectBounds = {
     left: number;
@@ -26,6 +81,12 @@ type PopmeltSubjectBounds = {
     width: number;
     height: number;
 };
+type PopmeltSubjectTrackingAnchor = {
+    /** Horizontal position normalized within the subject bounds. */
+    x: number;
+    /** Vertical position normalized within the subject bounds. */
+    y: number;
+};
 type PopmeltSubject = {
     key: string;
     label: string;
@@ -37,6 +98,9 @@ type PopmeltSubject = {
     /** Live owning surface; never serialized or included in model context. */
     sourceElement?: Element;
     imageObject?: ImageObjectContext;
+    videoObject?: VideoObjectContext;
+    /** Optional provider-neutral foreground anchors used by motion tracking. */
+    trackingAnchors?: readonly PopmeltSubjectTrackingAnchor[];
 };
 type PopmeltSurfaceSnapshot = {
     element: HTMLCanvasElement;
@@ -58,12 +122,16 @@ type PopmeltSubjectCreationSession = {
 type PopmeltSubjectAdapter = {
     id: string;
     /** Surface ownership; an explicit image adapter replaces Core's default. */
-    surfaceKind?: 'image';
+    surfaceKind?: 'image' | 'video';
+    /** Resolve once the adapter's persisted subject index is available. */
+    whenReady?(): Promise<void>;
     hitTest(clientPoint: {
         x: number;
         y: number;
     }): PopmeltSubject | null;
     resolve(key: string): PopmeltSubject | null;
+    /** Recreate a missing durable subject from annotation-captured context. */
+    restoreSubject?(key: string, elementInfo: ElementInfo): PopmeltSubject | null;
     /** Optional clean subject crop, without Popmelt's overlay. */
     captureSubject?(key: string): Blob | null | Promise<Blob | null>;
     /** Optional bitmap capture for renderer surfaces that DOM cloning cannot read. */
@@ -77,6 +145,8 @@ type PopmeltSubjectAdapter = {
     renameSubject?(key: string, label: string): PopmeltSubject | null;
     /** Replace a subject's viewport bounds, projected back into adapter space. */
     updateSubjectBounds?(key: string, bounds: PopmeltSubjectBounds): PopmeltSubject | null;
+    /** Replace optional foreground anchors normalized within subject bounds. */
+    updateSubjectTrackingAnchors?(key: string, anchors: readonly PopmeltSubjectTrackingAnchor[]): PopmeltSubject | null;
     /** Remove a user-created subject or hide an adapter-owned subject. */
     removeSubject?(key: string): void;
 };

package/dist/adapter-three.mjs

@@ -1 +1 @@
-import{Box3 as y,Raycaster as x,Vector2 as O,Vector3 as j}from"three";function b(e){return[e.x,e.y,e.z].map(n=>Number(n.toFixed(4))).join(", ")}function h(e){let n=e.material;return n&&(Array.isArray(n)?n:[n]).map(r=>r.name?.trim()||r.type).filter(Boolean).join(", ")||void 0}function M(e,n,c){let s=e.domElement.width,r=e.domElement.height;if(s<=0||r<=0)return null;let l=e.getRenderTarget(),t=e.xr.enabled;try{return e.xr.enabled=!1,e.setRenderTarget(null),e.render(n,c),e.domElement.toDataURL("image/png")}catch{return null}finally{e.setRenderTarget(l),e.xr.enabled=t}}function D(e){let s=(e.parent?.children??[]).filter(t=>t.type===e.type&&t.name===e.name),r=Math.max(0,s.indexOf(e)),l=e.name.trim()||e.type;return`${encodeURIComponent(l)}[${r}]`}function f(e){let n=[],c=e;for(;c?.parent;)n.push(D(c)),c=c.parent;return n.reverse().join("/")}function v(e){let n=e.userData.popmeltId;return typeof n=="string"&&n.trim()?n.trim():f(e)}function T(e){let n=e.userData.popmeltLabel;return typeof n=="string"&&n.trim()?n.trim():e.name.trim()||e.type}function R(e){let n=e;for(;n.parent;){let c=n.name.trim(),s=c.match(/^(.*)_(?:alpha|normal|material)(?:_\d+)+$/i),r=n.parent;for(;r.parent&&!r.name.trim();)r=r.parent;let l=r.name.trim();if(!l||!c.startsWith(`${l}_`)&&!s)break;n=r}return n}function P(e,n,c){let s=new y().setFromObject(e,!0);if(s.isEmpty())return null;let r=c.getBoundingClientRect(),l=[];for(let p of[s.min.x,s.max.x])for(let d of[s.min.y,s.max.y])for(let g of[s.min.z,s.max.z])l.push(new j(p,d,g).project(n));let t=l.map(p=>r.left+(p.x+1)/2*r.width),a=l.map(p=>r.top+(1-p.y)/2*r.height),i=Math.max(r.left,Math.min(...t)),o=Math.min(r.right,Math.max(...t)),m=Math.max(r.top,Math.min(...a)),u=Math.min(r.bottom,Math.max(...a));return o<=i||u<=m?null:{left:i,top:m,width:o-i,height:u-m}}function C(e){let n=e.raycaster??new x,c=new O,s=t=>e.key?.(t)??v(t),r=(t,a)=>{let i=s(t);if(!i)return null;let o=P(t,e.camera,e.canvas);return o?{key:i,label:e.label?.(t)??T(t),bounds:o,context:f(t),data:{renderer:"Three.js",objectType:t.type,...t.name?{objectName:t.name}:{},scenePath:f(t),...a&&a!==t?{renderObjectName:a.name.trim()||a.type,renderObjectType:a.type,renderScenePath:f(a)}:{},position:b(t.position),rotation:b(t.rotation),scale:b(t.scale),visible:String(t.visible),...h(t)?{material:h(t)}:{},...Object.fromEntries(Object.entries(e.describe?.(t)??{}).filter(m=>typeof m[1]=="string"&&m[1].length>0))}}:null},l=t=>{let a=null;return e.scene.traverse(i=>{!a&&s(i)===t&&(a=i)}),a};return{id:"three",hitTest({x:t,y:a}){let i=e.canvas.getBoundingClientRect();if(t<i.left||t>i.right||a<i.top||a>i.bottom)return null;c.set((t-i.left)/i.width*2-1,-((a-i.top)/i.height)*2+1),n.setFromCamera(c,e.camera);for(let o of n.intersectObject(e.scene,!0)){let m=e.select?e.select(o.object):R(o.object);if(!m||!m.visible)continue;let u=r(m,o.object);if(u)return u}return null},resolve(t){let a=l(t);return a?r(a):null},...e.captureFrame?{async captureSurfaces(){let t=await e.captureFrame?.();return t?[{element:e.canvas,dataUrl:t}]:[]}}:{}}}export{M as captureThreeRendererFrame,C as createThreeSubjectAdapter,R as selectThreeNamedPrimitiveParent,f as threeObjectPath};
+import{Box3 as y,Raycaster as x,Vector2 as O,Vector3 as j}from"three";function b(e){return[e.x,e.y,e.z].map(n=>Number(n.toFixed(4))).join(", ")}function h(e){let n=e.material;return n&&(Array.isArray(n)?n:[n]).map(r=>r.name?.trim()||r.type).filter(Boolean).join(", ")||void 0}function M(e,n,c){let s=e.domElement.width,r=e.domElement.height;if(s<=0||r<=0)return null;let l=e.getRenderTarget(),t=e.xr.enabled;try{return e.xr.enabled=!1,e.setRenderTarget(null),e.render(n,c),e.domElement.toDataURL("image/png")}catch{return null}finally{e.setRenderTarget(l),e.xr.enabled=t}}function D(e){let s=(e.parent?.children??[]).filter(t=>t.type===e.type&&t.name===e.name),r=Math.max(0,s.indexOf(e)),l=e.name.trim()||e.type;return`${encodeURIComponent(l)}[${r}]`}function f(e){let n=[],c=e;for(;c?.parent;)n.push(D(c)),c=c.parent;return n.reverse().join("/")}function v(e){let n=e.userData.popmeltId;return typeof n=="string"&&n.trim()?n.trim():f(e)}function T(e){let n=e.userData.popmeltLabel;return typeof n=="string"&&n.trim()?n.trim():e.name.trim()||e.type}function R(e){let n=e;for(;n.parent;){let c=n.name.trim(),s=c.match(/^(.*)_(?:alpha|normal|material)(?:_\d+)+$/i),r=n.parent;for(;r.parent&&!r.name.trim();)r=r.parent;let l=r.name.trim();if(!l||!c.startsWith(`${l}_`)&&!s)break;n=r}return n}function P(e,n,c){let s=new y().setFromObject(e,!0);if(s.isEmpty())return null;let r=c.getBoundingClientRect(),l=[];for(let p of[s.min.x,s.max.x])for(let d of[s.min.y,s.max.y])for(let g of[s.min.z,s.max.z])l.push(new j(p,d,g).project(n));let t=l.map(p=>r.left+(p.x+1)/2*r.width),a=l.map(p=>r.top+(1-p.y)/2*r.height),i=Math.max(r.left,Math.min(...t)),u=Math.min(r.right,Math.max(...t)),m=Math.max(r.top,Math.min(...a)),o=Math.min(r.bottom,Math.max(...a));return u<=i||o<=m?null:{left:i,top:m,width:u-i,height:o-m}}function C(e){let n=e.raycaster??new x,c=new O,s=t=>e.key?.(t)??v(t),r=(t,a)=>{let i=s(t);if(!i)return null;let u=P(t,e.camera,e.canvas);return u?{key:i,label:e.label?.(t)??T(t),sourceElement:e.canvas,bounds:u,context:f(t),data:{renderer:"Three.js",objectType:t.type,...t.name?{objectName:t.name}:{},scenePath:f(t),...a&&a!==t?{renderObjectName:a.name.trim()||a.type,renderObjectType:a.type,renderScenePath:f(a)}:{},position:b(t.position),rotation:b(t.rotation),scale:b(t.scale),visible:String(t.visible),...h(t)?{material:h(t)}:{},...Object.fromEntries(Object.entries(e.describe?.(t)??{}).filter(m=>typeof m[1]=="string"&&m[1].length>0))}}:null},l=t=>{let a=null;return e.scene.traverse(i=>{!a&&s(i)===t&&(a=i)}),a};return{id:"three",hitTest({x:t,y:a}){let i=e.canvas.getBoundingClientRect();if(t<i.left||t>i.right||a<i.top||a>i.bottom)return null;c.set((t-i.left)/i.width*2-1,-((a-i.top)/i.height)*2+1),n.setFromCamera(c,e.camera);for(let u of n.intersectObject(e.scene,!0)){let m=e.select?e.select(u.object):R(u.object);if(!m||!m.visible)continue;let o=r(m,u.object);if(o)return o}return null},resolve(t){let a=l(t);return a?r(a):null},...e.captureFrame?{async captureSurfaces(){let t=await e.captureFrame?.();return t?[{element:e.canvas,dataUrl:t}]:[]}}:{}}}export{M as captureThreeRendererFrame,C as createThreeSubjectAdapter,R as selectThreeNamedPrimitiveParent,f as threeObjectPath};