@myzonerocks/gosslens 0.10.0 → 0.11.0

package/dist/src/mic-input.d.ts

@@ -1,4 +1,4 @@
-import type { GossSession } from "./index";
+import type { GossSession } from "./index.js";
 export declare class GossMicInput {
     private session;
     private context;

package/dist/src/mic-input.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"mic-input.d.ts","sourceRoot":"","sources":["../../src/mic-input.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAiB3C,qBAAa,YAAY;IACvB,OAAO,CAAC,OAAO,CAAc;IAC7B,OAAO,CAAC,OAAO,CAA6B;IAC5C,OAAO,CAAC,MAAM,CAA4B;IAC1C,OAAO,CAAC,IAAI,CAAiC;IAC7C,OAAO,CAAC,MAAM,CAA2C;IAEzD,YAAY,OAAO,EAAE,WAAW,EAE/B;IAKK,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAuB3B;IAGK,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAU1B;CACF"}
+{"version":3,"file":"mic-input.d.ts","sourceRoot":"","sources":["../../src/mic-input.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAiB9C,qBAAa,YAAY;IACvB,OAAO,CAAC,OAAO,CAAc;IAC7B,OAAO,CAAC,OAAO,CAA6B;IAC5C,OAAO,CAAC,MAAM,CAA4B;IAC1C,OAAO,CAAC,IAAI,CAAiC;IAC7C,OAAO,CAAC,MAAM,CAA2C;IAEzD,YAAY,OAAO,EAAE,WAAW,EAE/B;IAKK,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAuB3B;IAGK,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAU1B;CACF"}

package/dist/src/video-texture.d.ts

@@ -1,4 +1,4 @@
-import type { GossSession } from "./index";
+import type { GossSession } from "./index.js";
 export declare class GossVideoTexture {
     private session;
     private name;

package/dist/src/video-texture.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"video-texture.d.ts","sourceRoot":"","sources":["../../src/video-texture.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAM3C,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,OAAO,CAAc;IAC7B,OAAO,CAAC,IAAI,CAAS;IACrB,OAAO,CAAC,KAAK,CAAmB;IAChC,OAAO,CAAC,MAAM,CAAoB;IAClC,OAAO,CAAC,GAAG,CAAkC;IAC7C,OAAO,CAAC,OAAO,CAAS;IAExB,YAAY,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAU1D;IAIK,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAK3B;IAKD,IAAI,IAAI,IAAI,CAYX;IAGD,IAAI,IAAI,IAAI,CAKX;CACF"}
+{"version":3,"file":"video-texture.d.ts","sourceRoot":"","sources":["../../src/video-texture.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAM9C,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,OAAO,CAAc;IAC7B,OAAO,CAAC,IAAI,CAAS;IACrB,OAAO,CAAC,KAAK,CAAmB;IAChC,OAAO,CAAC,MAAM,CAAoB;IAClC,OAAO,CAAC,GAAG,CAAkC;IAC7C,OAAO,CAAC,OAAO,CAAS;IAExB,YAAY,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAU1D;IAIK,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAK3B;IAKD,IAAI,IAAI,IAAI,CAYX;IAGD,IAAI,IAAI,IAAI,CAKX;CACF"}

package/dist/src/world.d.ts

@@ -1,4 +1,4 @@
-import type { GossSession } from "./index";
+import type { GossSession } from "./index.js";
 interface XRRigidTransformLike {
     matrix: Float32Array;
 }

package/dist/src/world.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"world.d.ts","sourceRoot":"","sources":["../../src/world.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,WAAW,EAAwC,MAAM,SAAS,CAAC;AAIjF,UAAU,oBAAoB;IAC5B,MAAM,EAAE,YAAY,CAAC;CACtB;AAED,UAAU,UAAU;IAClB,gBAAgB,EAAE,YAAY,CAAC;CAChC;AAED,UAAU,gBAAgB;IACxB,SAAS,EAAE,oBAAoB,CAAC;IAChC,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,UAAU,WAAW;IACnB,UAAU,EAAE,OAAO,CAAC;CACrB;AAED,UAAU,YAAY;IACpB,WAAW,EAAE,OAAO,CAAC;CACtB;AAED,UAAU,UAAU;IAClB,SAAS,EAAE,oBAAoB,CAAC;CACjC;AAED,MAAM,WAAW,eAAe;IAC9B,aAAa,CAAC,cAAc,EAAE,OAAO,GAAG,gBAAgB,GAAG,IAAI,CAAC;IAChE,OAAO,CAAC,KAAK,EAAE,OAAO,EAAE,cAAc,EAAE,OAAO,GAAG,UAAU,GAAG,IAAI,CAAC;IACpE,cAAc,CAAC,EAAE,GAAG,CAAC,WAAW,CAAC,CAAC;IAClC,cAAc,CAAC,EAAE,GAAG,CAAC,YAAY,CAAC,CAAC;CACpC;AAED,qBAAa,oBAAoB;IAKnB,OAAO,CAAC,QAAQ,CAAC,OAAO;IAJpC,OAAO,CAAC,QAAQ,CAAkC;IAClD,OAAO,CAAC,SAAS,CAAmC;IACpD,OAAO,CAAC,MAAM,CAAK;IAEnB,YAA6B,OAAO,EAAE,WAAW,EAAI;IAIrD,OAAO,CAAC,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,GAAG,IAAI,CA6ClF;IAED,OAAO,CAAC,KAAK;CAQd"}
+{"version":3,"file":"world.d.ts","sourceRoot":"","sources":["../../src/world.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,WAAW,EAAwC,MAAM,YAAY,CAAC;AAIpF,UAAU,oBAAoB;IAC5B,MAAM,EAAE,YAAY,CAAC;CACtB;AAED,UAAU,UAAU;IAClB,gBAAgB,EAAE,YAAY,CAAC;CAChC;AAED,UAAU,gBAAgB;IACxB,SAAS,EAAE,oBAAoB,CAAC;IAChC,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,UAAU,WAAW;IACnB,UAAU,EAAE,OAAO,CAAC;CACrB;AAED,UAAU,YAAY;IACpB,WAAW,EAAE,OAAO,CAAC;CACtB;AAED,UAAU,UAAU;IAClB,SAAS,EAAE,oBAAoB,CAAC;CACjC;AAED,MAAM,WAAW,eAAe;IAC9B,aAAa,CAAC,cAAc,EAAE,OAAO,GAAG,gBAAgB,GAAG,IAAI,CAAC;IAChE,OAAO,CAAC,KAAK,EAAE,OAAO,EAAE,cAAc,EAAE,OAAO,GAAG,UAAU,GAAG,IAAI,CAAC;IACpE,cAAc,CAAC,EAAE,GAAG,CAAC,WAAW,CAAC,CAAC;IAClC,cAAc,CAAC,EAAE,GAAG,CAAC,YAAY,CAAC,CAAC;CACpC;AAED,qBAAa,oBAAoB;IAKnB,OAAO,CAAC,QAAQ,CAAC,OAAO;IAJpC,OAAO,CAAC,QAAQ,CAAkC;IAClD,OAAO,CAAC,SAAS,CAAmC;IACpD,OAAO,CAAC,MAAM,CAAK;IAEnB,YAA6B,OAAO,EAAE,WAAW,EAAI;IAIrD,OAAO,CAAC,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,GAAG,IAAI,CA6ClF;IAED,OAAO,CAAC,KAAK;CAQd"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@myzonerocks/gosslens",
-  "version": "0.10.0",
+  "version": "0.11.0",
   "description": "Web SDK for the gosslens camera and AR engine, over the wasm core.",
   "license": "Apache-2.0",
   "repository": {

package/src/audio-output.ts

@@ -1,4 +1,4 @@
-import type { GossSession } from "./index";
+import type { GossSession } from "./index.js";
 
 /// The worklet drains chunks the page posts it, so the audio thread
 /// never touches the wasm heap and the engine's graph-thread pull

package/src/index.ts

@@ -163,6 +163,28 @@ export interface GossPoseOut {
 
 export const GOSS_HAND_LANDMARK_COUNT = 21;
 export const GOSS_HAND_MAX = 2;
+
+export const GOSS_CAP_TRACKING = 1 << 0;
+export const GOSS_CAP_SEGMENTATION = 1 << 1;
+export const GOSS_CAP_ML_INFER = 1 << 2;
+export const GOSS_CAP_DIFFUSION = 1 << 3;
+export const GOSS_CAP_BEAUTY = 1 << 4;
+export const GOSS_CAP_PHYSICS = 1 << 5;
+export const GOSS_CAP_VIDEO_TEXTURES = 1 << 6;
+export const GOSS_CAP_PHOTO_CAPTURE = 1 << 7;
+export const GOSS_CAP_RECORDING = 1 << 8;
+export const GOSS_CAP_FILE_IO = 1 << 9;
+
+/// A hand handed to submitHands: the tracker's presence, handedness, gesture,
+/// and GOSS_HAND_LANDMARK_COUNT * 3 frame-pixel landmark floats. The shape
+/// GossHandTracker.process returns plugs in directly.
+export interface GossHandInput {
+  presence?: number;
+  handedness?: number;
+  gesture?: number;
+  gestureScore?: number;
+  landmarks: Float32Array | number[];
+}
 const HAND_RESULT_BYTES = 560;
 const HAND_ONE_BYTES = 268;
 
@@ -371,6 +393,13 @@ export class Gosslens {
     return this.version;
   }
 
+  /// Which capabilities this build compiled real, as GOSS_CAP_* bits. A stub
+  /// build shares the full one's filename and abi version, so check the rail
+  /// you need here before feeding real bytes to an enable call.
+  capabilities(): number {
+    return this.mod.ccall("goss_capabilities", "number", [], []) as number;
+  }
+
   /// Loads gosslens_web.js and checks its ABI major version. A
   /// dynamic import, not static: bun's bundler would otherwise inline
   /// this file, breaking Emscripten's own import.meta.url-relative
@@ -1635,7 +1664,7 @@ export class GossSession {
     this.mod.ccall("goss_session_brush_clear", "number", ["number"], [this.handle]);
   }
 
-  /// The brush preset the next stroke opens with: 0 pen, 1 highlighter, 2 marker, 3 neon.
+  /// The brush preset the next stroke opens with: 0 pen, 1 highlighter, 2 marker, 3 neon, 4 stamp.
   setBrushMode(mode: number): void {
     this.mod.ccall("goss_session_brush_set_mode", "number", ["number", "number"], [this.handle, mode]);
   }
@@ -1879,6 +1908,34 @@ export class GossSession {
     this.mod.ccall("goss_session_submit_bodies", "number", ["number", "number", "number"], [this.handle, ptr, bodies.length]);
   }
 
+  /// Submits the hands tracked this frame from the page's own tracker (the
+  /// GossHandTracker's result plugs in directly), so hands.gesture and
+  /// hands.pinch lenses and handJoint work with no in-engine worker; the
+  /// submitted hands win over the worker while set. Null clears the path.
+  submitHands(hands: { hands: GossHandInput[]; timestampUs?: bigint } | null): void {
+    if (hands === null) {
+      this.mod.ccall("goss_session_submit_hands", "number", ["number", "number"], [this.handle, 0]);
+      return;
+    }
+    const ptr = this.scratch(HAND_RESULT_BYTES);
+    this.mod.HEAPU8.fill(0, ptr, ptr + HAND_RESULT_BYTES);
+    const dv = new DataView(this.mod.HEAPU8.buffer, ptr, HAND_RESULT_BYTES);
+    dv.setBigInt64(8, hands.timestampUs ?? 0n, true);
+    const list = hands.hands.slice(0, GOSS_HAND_MAX);
+    dv.setUint32(16, list.length, true);
+    for (let i = 0; i < list.length; i += 1) {
+      const off = 24 + i * HAND_ONE_BYTES;
+      const h = list[i]!;
+      dv.setFloat32(off, h.presence ?? 1, true);
+      dv.setFloat32(off + 4, h.handedness ?? 0.5, true);
+      dv.setUint32(off + 8, h.gesture ?? 0, true);
+      dv.setFloat32(off + 12, h.gestureScore ?? 0, true);
+      const floats = Math.min(h.landmarks.length, GOSS_HAND_LANDMARK_COUNT * 3);
+      for (let k = 0; k < floats; k += 1) dv.setFloat32(off + 16 + k * 4, h.landmarks[k]!, true);
+    }
+    this.mod.ccall("goss_session_submit_hands", "number", ["number", "number"], [this.handle, ptr]);
+  }
+
   /// Submits one frame's depth map from the host AR backend (WebXR depth-
   /// sensing): width by height metres per pixel, row major, with the near and
   /// far metres that bound it. An empty array clears it. Kept for depth
@@ -2039,6 +2096,75 @@ export class GossSession {
     return result;
   }
 
+  /// Hands the engine one bundle asset's bytes under its manifest name ahead
+  /// of activateLens, so a lens's models, references, and label files load
+  /// from memory - the door the web's inference nodes activate through. An
+  /// empty array removes a staged name; the engine keeps its own copy.
+  provideLensAsset(name: string, bytes: Uint8Array): void {
+    const id = new TextEncoder().encode(name);
+    const idPtr = this.mod.ccall("goss_alloc", "number", ["number"], [id.length || 1]) as number;
+    this.mod.HEAPU8.set(id, idPtr);
+    let bytesPtr = 0;
+    if (bytes.length > 0) {
+      bytesPtr = this.mod.ccall("goss_alloc", "number", ["number"], [bytes.length]) as number;
+      this.mod.HEAPU8.set(bytes, bytesPtr);
+    }
+    const args = ["number", "number", "number", "number", "number"];
+    const status = this.mod.ccall("goss_session_provide_lens_asset", "number", args, [this.handle, idPtr, id.length, bytesPtr, bytes.length]) as number;
+    if (bytesPtr !== 0) this.mod.ccall("goss_free", null, ["number", "number"], [bytesPtr, bytes.length]);
+    this.mod.ccall("goss_free", null, ["number", "number"], [idPtr, id.length || 1]);
+    if (status !== 0) throw new Error(`provideLensAsset failed (${status})`);
+  }
+
+  /// One ml.infer node's whole published output tensor, by the node's id and
+  /// tensor index, or null before the model's first publish or when the node
+  /// or tensor does not exist. A length probe sizes the read, so detections,
+  /// embeddings, and logits come back at their own length.
+  mlOutput(nodeId: string, tensor = 0): Float32Array | null {
+    const id = new TextEncoder().encode(nodeId);
+    const idPtr = this.mod.ccall("goss_alloc", "number", ["number"], [id.length || 1]) as number;
+    this.mod.HEAPU8.set(id, idPtr);
+    const lenPtr = this.mod.ccall("goss_alloc", "number", ["number"], [4]) as number;
+    const readLen = () => new DataView(this.mod.HEAPU8.buffer, lenPtr, 4).getUint32(0, true);
+    const args = ["number", "number", "number", "number", "number", "number", "number"];
+    this.mod.ccall("goss_session_ml_output", "number", args, [this.handle, idPtr, id.length, tensor, 0, 0, lenPtr]);
+    const needed = readLen();
+    let result: Float32Array | null = null;
+    if (needed > 0) {
+      const outPtr = this.mod.ccall("goss_alloc", "number", ["number"], [needed * 4]) as number;
+      const status = this.mod.ccall("goss_session_ml_output", "number", args, [this.handle, idPtr, id.length, tensor, outPtr, needed, lenPtr]) as number;
+      if (status === 0) result = this.mod.HEAPF32.slice(outPtr >> 2, (outPtr >> 2) + needed);
+      this.mod.ccall("goss_free", null, ["number", "number"], [outPtr, needed * 4]);
+    }
+    this.mod.ccall("goss_free", null, ["number", "number"], [lenPtr, 4]);
+    this.mod.ccall("goss_free", null, ["number", "number"], [idPtr, id.length || 1]);
+    return result;
+  }
+
+  /// One ml.infer node's mask-bound output resampled to the fixed
+  /// segmentation plane, or null when the node has no mask binding or the
+  /// model has not published yet.
+  mlMask(nodeId: string): Float32Array | null {
+    const id = new TextEncoder().encode(nodeId);
+    const idPtr = this.mod.ccall("goss_alloc", "number", ["number"], [id.length || 1]) as number;
+    this.mod.HEAPU8.set(id, idPtr);
+    const lenPtr = this.mod.ccall("goss_alloc", "number", ["number"], [4]) as number;
+    const readLen = () => new DataView(this.mod.HEAPU8.buffer, lenPtr, 4).getUint32(0, true);
+    const args = ["number", "number", "number", "number", "number", "number"];
+    this.mod.ccall("goss_session_ml_mask", "number", args, [this.handle, idPtr, id.length, 0, 0, lenPtr]);
+    const needed = readLen();
+    let result: Float32Array | null = null;
+    if (needed > 0) {
+      const outPtr = this.mod.ccall("goss_alloc", "number", ["number"], [needed * 4]) as number;
+      const status = this.mod.ccall("goss_session_ml_mask", "number", args, [this.handle, idPtr, id.length, outPtr, needed, lenPtr]) as number;
+      if (status === 0) result = this.mod.HEAPF32.slice(outPtr >> 2, (outPtr >> 2) + needed);
+      this.mod.ccall("goss_free", null, ["number", "number"], [outPtr, needed * 4]);
+    }
+    this.mod.ccall("goss_free", null, ["number", "number"], [lenPtr, 4]);
+    this.mod.ccall("goss_free", null, ["number", "number"], [idPtr, id.length || 1]);
+    return result;
+  }
+
   /// The recent diarized caption segment at index (0 the newest), or null when
   /// the index is past the segments held: the times it spanned, the speaker who
   /// spoke it, and its text, a speaker-tagged transcript for diarized subtitles.
@@ -2852,8 +2978,8 @@ export class GossPreviewSession {
   }
 }
 
-export { GossWebXRWorldSource } from "./world";
-export type { GossXRFrameLike } from "./world";
-export { GossAudioOutput } from "./audio-output";
-export { GossMicInput } from "./mic-input";
-export { GossVideoTexture } from "./video-texture";
+export { GossWebXRWorldSource } from "./world.js";
+export type { GossXRFrameLike } from "./world.js";
+export { GossAudioOutput } from "./audio-output.js";
+export { GossMicInput } from "./mic-input.js";
+export { GossVideoTexture } from "./video-texture.js";

package/src/mic-input.ts

@@ -1,4 +1,4 @@
-import type { GossSession } from "./index";
+import type { GossSession } from "./index.js";
 
 /// Captures the microphone and feeds it to the engine's audio analysis,
 /// so the level and beat triggers a lens reads fire on web the same way

package/src/video-texture.ts

@@ -1,4 +1,4 @@
-import type { GossSession } from "./index";
+import type { GossSession } from "./index.js";
 
 /// Plays an MP4 through a hidden video element, the browser's own
 /// decoder, and feeds each frame to a named engine source a lens

package/src/world.ts

@@ -2,7 +2,7 @@
 /// planes, and tracked anchors off each XR frame and feeds them into
 /// the session. The page owns the XR session; this source only reads.
 
-import type { GossSession, GossWorldPlane, GossWorldAnchorInput } from "./index";
+import type { GossSession, GossWorldPlane, GossWorldAnchorInput } from "./index.js";
 
 // WebXR's types live outside the DOM lib; the narrow slice this source
 // reads is declared here so no type package enters the build.