@supersoniks/concorde 4.9.96 → 4.9.98-visual-stack.0

package/src/core/utils/mediaRef.ts

@@ -0,0 +1,193 @@
+/**
+ * Jeton média neutre pour chaîner HF → jsonata → shader / 3d via DataProvider.
+ *
+ * - `url` : http(s) | blob: | data: (préféré pour les canaux shader)
+ * - `blob` : binaire (HF, upload) — le producteur publie aussi `url` quand possible
+ * - `blobUrl` : alias historique (extract shader)
+ * - `element` : sélecteur `#id` ou Element (pont live → canaux shader, ignoré par toMediaUrl)
+ */
+
+export type SonicMediaRef = {
+  url?: string;
+  blobUrl?: string;
+  blob?: Blob;
+  width?: number;
+  height?: number;
+  mime?: string;
+  /** Sélecteur `#id` ou référence Element (upload live, pas d’URL). */
+  element?: string | Element;
+};
+
+/**
+ * Contrat minimal pour qu’un composant Concorde alimente un canal shader
+ * sans JPEG / blob (`channel0="#plateau"` ou `{ element }`).
+ */
+export type SonicFrameSource = {
+  /** Canvas lisible (preserveDrawingBuffer ou 2D). `null` tant que pas prêt. */
+  getFrameCanvas(): HTMLCanvasElement | null;
+  /** Incrémenté après chaque rendu effectif. */
+  readonly frameSeq: number;
+};
+
+export function isFrameSource(el: unknown): el is SonicFrameSource {
+  if (!el || typeof el !== "object") return false;
+  const o = el as SonicFrameSource;
+  return (
+    typeof o.getFrameCanvas === "function" && typeof o.frameSeq === "number"
+  );
+}
+
+/** Clés synthétiques pour Elements DP sans `id` (`#__sonic_frame_N`). */
+const LIVE_PREFIX = "#__sonic_frame_";
+let liveSeq = 0;
+const liveByKey = new Map<string, Element>();
+const keyByLive = new WeakMap<Element, string>();
+
+/** Enregistre un Element sans id → clé `#__sonic_frame_N` (réutilisée). */
+export function registerLiveElement(el: Element): string {
+  const existing = keyByLive.get(el);
+  if (existing) return existing;
+  const key = `${LIVE_PREFIX}${++liveSeq}`;
+  liveByKey.set(key, el);
+  keyByLive.set(el, key);
+  return key;
+}
+
+export function lookupLiveElement(key: string): Element | null {
+  if (!key.startsWith(LIVE_PREFIX)) return null;
+  const el = liveByKey.get(key) ?? null;
+  if (el && !el.isConnected) {
+    liveByKey.delete(key);
+    return null;
+  }
+  return el;
+}
+
+/** Normalise `element` DP → clé canal (`#id` ou live). */
+export function mediaElementKey(value: unknown): string | null {
+  if (value == null) return null;
+  if (typeof value === "string") {
+    const s = value.trim();
+    if (!s) return null;
+    return s.startsWith("#") ? s : `#${s}`;
+  }
+  if (typeof Element !== "undefined" && value instanceof Element) {
+    if (value.id) return `#${value.id}`;
+    return registerLiveElement(value);
+  }
+  return null;
+}
+
+/** Objet image sérialisé (sortie HF / extract) : Blob + dimensions. */
+export function isMediaBlobObject(
+  value: unknown,
+): value is SonicMediaRef & { blob: Blob } {
+  if (!value || typeof value !== "object") return false;
+  const o = value as Record<string, unknown>;
+  return (
+    o.blob instanceof Blob &&
+    (typeof o.width === "number" ||
+      typeof o.height === "number" ||
+      typeof o.channels === "number" ||
+      typeof o.url === "string" ||
+      typeof o.blobUrl === "string")
+  );
+}
+
+/**
+ * Résout une URL affichable / chargeable sans créer d’ObjectURL.
+ * Accepte : string | objet avec `url` / `blobUrl` | null.
+ * Ne matérialise pas un `blob` seul — utiliser `attachMediaUrls` / `cloneMediaRef`.
+ */
+function nonEmptyTrimmed(value: unknown): string | null {
+  if (typeof value !== "string") return null;
+  const trimmed = value.trim();
+  return trimmed || null;
+}
+
+export function toMediaUrl(value: unknown): string | null {
+  if (value == null || value === "") return null;
+  if (typeof value === "string") return nonEmptyTrimmed(value);
+  if (typeof value !== "object") return null;
+  const o = value as Record<string, unknown>;
+  return nonEmptyTrimmed(o.url) ?? nonEmptyTrimmed(o.blobUrl);
+}
+
+/**
+ * Attache un `url` (ObjectURL) à chaque nœud image `{ blob, width?, height? }`
+ * dans une arborescence. Mutates in place. Les URLs créées sont poussées dans
+ * `created` pour revoke ultérieur.
+ */
+export function attachMediaUrls(
+  value: unknown,
+  created: string[],
+): unknown {
+  if (Array.isArray(value)) {
+    for (let i = 0; i < value.length; i++) {
+      value[i] = attachMediaUrls(value[i], created);
+    }
+    return value;
+  }
+  if (!value || typeof value !== "object") return value;
+  const o = value as Record<string, unknown>;
+  if (isMediaBlobObject(o)) {
+    // Ne pas écraser une url http(s)/blob déjà fournie par le producteur.
+    if (toMediaUrl(o)) return value;
+    const url = URL.createObjectURL(o.blob);
+    created.push(url);
+    o.url = url;
+    return value;
+  }
+  for (const key of Object.keys(o)) {
+    o[key] = attachMediaUrls(o[key], created);
+  }
+  return value;
+}
+
+/** Révoque une liste d’ObjectURL (ignore les erreurs). */
+export function revokeMediaUrls(urls: Iterable<string>): void {
+  for (const url of urls) {
+    try {
+      URL.revokeObjectURL(url);
+    } catch {
+      /* ignore */
+    }
+  }
+}
+
+/**
+ * Copie durable d’un média : nouvel ObjectURL depuis `blob` (ou fetch de `url`).
+ * À utiliser quand on fige un `frameUrl` 3d — le producteur révoque sinon au frame suivant.
+ */
+export async function cloneMediaRef(
+  value: unknown,
+): Promise<SonicMediaRef | null> {
+  if (value == null) return null;
+  if (typeof value === "string") {
+    const url = value.trim();
+    if (!url) return null;
+    try {
+      const res = await fetch(url);
+      const blob = await res.blob();
+      const cloned = URL.createObjectURL(blob);
+      return { url: cloned, blob, mime: blob.type || undefined };
+    } catch {
+      return null;
+    }
+  }
+  if (typeof value !== "object") return null;
+  const o = value as SonicMediaRef;
+  if (o.blob instanceof Blob) {
+    const url = URL.createObjectURL(o.blob);
+    return {
+      url,
+      blob: o.blob,
+      width: o.width,
+      height: o.height,
+      mime: o.mime || o.blob.type || undefined,
+    };
+  }
+  const existing = toMediaUrl(o);
+  if (!existing) return null;
+  return cloneMediaRef(existing);
+}

package/src/core/utils/similarity.spec.ts

@@ -0,0 +1,34 @@
+import { describe, expect, it } from "vitest";
+import { cosine, rankBySimilarity } from "@supersoniks/concorde/core/utils/similarity";
+import { compileJsonata } from "@supersoniks/concorde/core/utils/jsonataRuntime";
+
+describe("similarity", () => {
+  it("cosine", () => {
+    expect(cosine([1, 0], [1, 0])).toBe(1);
+    expect(cosine([1, 0], [0, 1])).toBe(0);
+    expect(cosine([1, 0], [1, 0, 0])).toBeNull();
+    expect(cosine(null, [1])).toBeNull();
+  });
+
+  it("rankBySimilarity : classe, ignore les vecteurs absents, coupe au top N", () => {
+    const items = [{ id: "a" }, { id: "b" }, { id: "c" }];
+    const vectors = [[0, 1], null, [1, 0.1]];
+    const ranked = rankBySimilarity(items, vectors, [1, 0], 5);
+    expect(ranked.map((r) => r.id)).toEqual(["c", "a"]);
+    expect(ranked[0]._score).toBeGreaterThan(0.99);
+    expect(rankBySimilarity(items, vectors, [1, 0], 1)).toHaveLength(1);
+  });
+
+  it("disponible en JSONata ($rankBySimilarity, $cosine)", async () => {
+    const expr = compileJsonata(
+      "$rankBySimilarity($input.items, $input.vectors, $input.query, 1).id",
+    );
+    const result = await expr.evaluate(
+      {},
+      { input: { items: [{ id: "x" }, { id: "y" }], vectors: [[1, 0], [0, 1]], query: [0, 1] } },
+    );
+    expect(result).toBe("y");
+    const cos = await compileJsonata("$cosine([1,0],[1,0])").evaluate({}, {});
+    expect(cos).toBe(1);
+  });
+});

package/src/core/utils/similarity.ts

@@ -0,0 +1,54 @@
+/**
+ * Similarité entre vecteurs (embeddings). Fonctions pures, sans dépendance :
+ * utilisables en TypeScript et exposées à JSONata (`$cosine`, `$rankBySimilarity`).
+ */
+
+function isVector(value: unknown): value is number[] {
+  return (
+    Array.isArray(value) &&
+    value.length > 0 &&
+    value.every((v) => typeof v === "number" && Number.isFinite(v))
+  );
+}
+
+/** Similarité cosinus dans [-1, 1] ; `null` si les vecteurs sont absents ou incompatibles. */
+export function cosine(a: unknown, b: unknown): number | null {
+  if (!isVector(a) || !isVector(b) || a.length !== b.length) return null;
+  let dot = 0;
+  let na = 0;
+  let nb = 0;
+  for (let i = 0; i < a.length; i++) {
+    dot += a[i] * b[i];
+    na += a[i] * a[i];
+    nb += b[i] * b[i];
+  }
+  if (!na || !nb) return null;
+  return dot / Math.sqrt(na * nb);
+}
+
+export type Ranked<T> = T & { _score: number };
+
+/**
+ * Classe `items` par similarité entre leur vecteur (`vectors[i]`, aligné sur
+ * `items`) et `query`. Les éléments sans vecteur sont ignorés.
+ */
+export function rankBySimilarity<T>(
+  items: readonly T[] | null | undefined,
+  vectors: readonly (readonly number[] | null)[] | null | undefined,
+  query: readonly number[] | null | undefined,
+  topN = 10,
+  minScore = -1,
+): Ranked<T>[] {
+  if (!Array.isArray(items) || !Array.isArray(vectors) || !isVector(query))
+    return [];
+  const scored: Ranked<T>[] = [];
+  items.forEach((item, i) => {
+    const score = cosine(vectors[i], query);
+    if (score === null || score < minScore) return;
+    const base =
+      item && typeof item === "object" ? item : ({ value: item } as unknown as T);
+    scored.push({ ...(base as T), _score: Math.round(score * 10_000) / 10_000 });
+  });
+  scored.sort((a, b) => b._score - a._score);
+  return scored.slice(0, Math.max(0, topN));
+}

package/src/docs/_core-concept/dataFlow.md

@@ -19,6 +19,8 @@ Recommended patterns for new Concorde apps (Lit + TypeScript). Under the hood, d
 | Derived / aggregate (JSONata) | `derive()` or `sonic-jsonata` — [Jsonata](#core/components/functional/jsonata/jsonata.md/jsonata) |
 | Offline doc demos | `serviceURL="/docs-mock-api"` — [Local API demos](#docs/_misc/docs-mock-api.md/docs-mock-api) |
 
+Opt-in addons (HF / shader / WebGPU): [Visual stack](#docs/_misc/visual-stack.md/visual-stack).
+
 Skill: `concorde-get-set-dp` in the package `ai/` folder.
 
 ## DataProviderKey

package/src/docs/_getting-started/cdn-addons.md

@@ -0,0 +1,73 @@
+# CDN addons
+
+Heavy opt-in features stay **out of** `concorde-core.bundle.js`. Load the core bundle first, then the addon you need.
+
+| Addon | Bundle | Custom element | Notes |
+|-------|--------|----------------|-------|
+| 3D | `concorde-3d.bundle.js` | `sonic-3d` | Includes Three.js |
+| Shader | `concorde-shader.bundle.js` | `sonic-shader` | WebGL2 / ShaderToy |
+| WebGPU | `concorde-webgpu.bundle.js` | `sonic-webgpu` | Compute (`particles`) + image WGSL; DP / MediaRef glue same as shader |
+| Hugging Face | `concorde-hugging-face-infer.bundle.js` | `sonic-hugging-face-infer` | Exposes `ConcordeHF` for config |
+
+npm: `@supersoniks/concorde/bundle/3d`, `…/bundle/shader`, `…/bundle/webgpu`, `…/bundle/hugging-face-infer` (IIFE). ESM twins: `…/bundle/3d/es`, etc.
+
+## Script tags
+
+<sonic-code language="html">
+  <template>
+&lt;script src="https://cdn.example.org/concorde/concorde-core.bundle.js"&gt;&lt;/script&gt;
+&lt;script src="https://cdn.example.org/concorde/concorde-shader.bundle.js"&gt;&lt;/script&gt;
+&lt;sonic-shader image-ch0="/photo.jpg" image-shader="…"&gt;&lt;/sonic-shader&gt;
+  </template>
+</sonic-code>
+
+## SDUI (`js[]`)
+
+Scripts in the descriptor are **awaited** before nodes are created. A failed `loadJS` **throws** (no UI parse without deps). Prefer the IIFE `.bundle.js` files. Descriptors (`js[]`, `markup`, `innerHTML`) must be **trusted**.
+
+<sonic-code language="html">
+  <template>
+&lt;sonic-sdui props='{
+  "js": ["https://cdn.example.org/concorde/concorde-3d.bundle.js"],
+  "nodes": [{
+    "tagName": "sonic-3d",
+    "attributes": { "src": "https://cdn.jsdelivr.net/gh/KhronosGroup/glTF-Sample-Models@master/2.0/DamagedHelmet/glTF-Binary/DamagedHelmet.glb" }
+  }]
+}'&gt;&lt;/sonic-sdui&gt;
+  </template>
+</sonic-code>
+
+## Hugging Face — minimal bootstrap
+
+The CE alone is not enough: configure runtime + allowlist once (page script or app entry). The IIFE exposes `window.ConcordeHF`.
+
+<sonic-code language="html">
+  <template>
+&lt;script src="…/concorde-core.bundle.js"&gt;&lt;/script&gt;
+&lt;script src="…/concorde-hugging-face-infer.bundle.js"&gt;&lt;/script&gt;
+&lt;script&gt;
+  ConcordeHF.configureHuggingFace({
+    runtime: {
+      source: "cdn",
+      version: "4.3.0",
+      url: "https://cdn.example.org/vendor/hf-transformers/4.3.0/transformers.min.js",
+    },
+    models: ConcordeHF.defineHuggingFaceModels({
+      "multilingual-mini": {
+        task: "feature-extraction",
+        repo: "Xenova/paraphrase-multilingual-MiniLM-L12-v2",
+        revision: "&lt;sha&gt;",
+        options: { dtype: "q8" },
+      },
+    }),
+  });
+&lt;/script&gt;
+&lt;sonic-hugging-face-infer
+  model="multilingual-mini"
+  inputProvider="events"
+  outputProvider="search.vectors"
+&gt;&lt;/sonic-hugging-face-infer&gt;
+  </template>
+</sonic-code>
+
+Peer deps (npm, optional): `three@0.180.0`, `@huggingface/transformers@4.3.0`. Not needed when using the CDN IIFE addons (Three is inlined in `concorde-3d`; Transformers.js stays on its own CDN via `configureHuggingFace`).

package/src/docs/_getting-started/create-a-component.md

@@ -113,7 +113,7 @@ This leads to the creation of more components and thus raises questions about th
 
 To compile the component, it needs to be referenced somewhere through an import statement. In particular, it is important to reference it in any component that uses it.
 
-In the case where it can be directly used in a page, it should also be globally referenced, especially considering the creation of **specific bundles** in the future.
+In the case where it can be directly used in a page, it should also be globally referenced. Heavy / optional features (3d, shader, webgpu, hugging-face) use **CDN addon bundles** — see [CDN addons](#docs/_getting-started/cdn-addons.md/cdn-addons) — and must **not** be added to `functional.ts` / `ui.ts`.
 
 Here's where we add imports based on the component's location inside concorde as an example
 

package/src/docs/_misc/visual-stack.md

@@ -0,0 +1,65 @@
+# Visual stack
+
+Examples of chaining opt-in addons (HF, shader, WebGPU, 3D). Outside the main “functional” flow.
+
+## HF depth → parallax
+
+Photo → `sonic-hugging-face-infer` (depth) → `sonic-shader` (parallax) → extract → `sonic-jsonata` (stats / thresholds).
+
+### Golden rule
+
+| Path | Role |
+|------|------|
+| **Hot** (textures / uniforms) | Direct DP: photo + depth → `fx` → shader (`param0` via form) |
+| **Cold** (aggregates) | `sonic-jsonata` **after** extract — counts, averages, suggested thresholds |
+
+Do not pipe texture channels through jsonata on every frame / slider: that forces rebuilds. JSONata is for **scalar derivatives**, not the media bus.
+
+### Pipeline
+
+```text
+photo (sample)
+    → sonic-hugging-face-infer depth (trigger=demand)
+    → set fx { bufferACh0, imageCh1, param0 }   ← direct wiring
+    → sonic-shader parallax + extract
+    → sonic-jsonata → { count, avgScore, strong, suggestedThreshold }
+```
+
+### Demo
+
+<docs-lit-demo for="docs-visual-stack-demo"></docs-lit-demo>
+
+## 3D → shader live (GPU bridge)
+
+`sonic-3d` and `sonic-shader` keep separate WebGL contexts. For realtime post, point a channel at the 3D host with `#id` — the shader copies the canvas each frame (`texImage2D`), no JPEG.
+
+```html
+<sonic-3d id="plateau" play auto-rotate …></sonic-3d>
+<sonic-shader channel0="#plateau" image="…" play></sonic-shader>
+```
+
+Also accepted: `<img>`, `<video>`, `<canvas>`, or another Concorde host that implements `SonicFrameSource` (`getFrameCanvas` + `frameSeq`). DP form: `{ element: "#plateau" }` or `{ element: el }`.
+
+`frame-out` (JPEG ObjectURL) remains for export / HF / freeze — not for live post.
+
+<docs-lit-demo for="docs-3d-shader-live-demo"></docs-lit-demo>
+
+## Harris → patches → WebGPU
+
+`sonic-shader` detects corners (Harris) and extracts patches → `channel0…3` on `sonic-webgpu` (`kernel="image"`): GPU mosaic.
+
+<docs-lit-demo for="docs-harris-webgpu-demo"></docs-lit-demo>
+
+## Components
+
+| Step | Doc |
+|------|-----|
+| Depth | [Hugging Face infer](#core/components/functional/infer/hugging-face/hugging-face-infer.md/hugging-face-infer) |
+| FX + extract | [Shader](#core/components/functional/shader/shader.md/shader) |
+| 3D viewer | [3D](#core/components/functional/3d/3d.md/3d) |
+| Compute / mosaic | [WebGPU](#core/components/functional/webgpu/webgpu.md/webgpu) |
+| Stats / thresholds | [Jsonata](#core/components/functional/jsonata/jsonata.md/jsonata) |
+
+## Media contract
+
+`SonicMediaRef` / `SonicFrameSource` / `toMediaUrl` / `cloneMediaRef` — `@supersoniks/concorde/core/utils/mediaRef`.