@widgetic/creator 0.3.71 → 0.3.73

package/dist/utils/designIntentTelemetry.js

@@ -0,0 +1,474 @@
+/**
+ * designIntentTelemetry — shadow-mode measurement for the canvas chat.
+ *
+ * Every message a user sends in the canvas chat is bucketed by *how it could be
+ * resolved*, independently of what the app actually does:
+ *
+ *   literal_arithmetic   — explicit value / arithmetic in code (the deterministic
+ *                          resolver already handles it; no model needed)
+ *   bounded_set_semantic — a design ask whose property/value needs judgment
+ *                          (the JEV-shaped residual: Choice over the closed set)
+ *   free_text            — needs a text value (an LLM)
+ *   new_property         — asks for a design property that does not exist yet
+ *                          (generation; an LLM)
+ *   not_design           — not a design tweak at all → codegen as today
+ *
+ * This module is measurement only: it never applies a change and never alters a
+ * send. It records one event per message, persists rolling counts so fractions
+ * survive reloads, and reports them. Enabling it is safe in production.
+ *
+ * The Jev fallback is measured twice over: how big its candidate pool is (the
+ * bounded-set-semantic bucket) and how much of that pool it resolves. Every dial
+ * is attributed to the *ask shape* that produced it, so a fall-through count can
+ * be read against the question it implicates — a weak value bucket and a weak
+ * property roster produce entirely different reasons — instead of as one global
+ * total.
+ */
+import { collectDesignProperties, resolveDesignIntent, scoreDesignProperties } from './designIntentResolver';
+export const DESIGN_INTENT_BUCKETS = [
+    'literal_arithmetic',
+    'bounded_set_semantic',
+    'free_text',
+    'new_property',
+    'not_design'
+];
+export const JE_ASK_SHAPES = ['property_named', 'value_only', 'unknown'];
+const TEXT_CONTROLLERS = new Set(['textinput', 'textarea']);
+/**
+ * Words that signal a design ask even when no schema property is named — value
+ * words (colors, sizes) plus the semantic adjectives users reach for when they
+ * want a look rather than a value ("make it pop", "more breathing room").
+ */
+const DESIGN_VALUE_RE = new RegExp('\\b(' +
+    [
+        'rounder', 'rounded', 'radius', 'corner', 'corners', 'darker', 'lighter',
+        'brighter', 'bigger', 'smaller', 'wider', 'narrower', 'taller', 'shorter',
+        'padding', 'spacing', 'margin', 'gap', 'color', 'colour', 'tint',
+        'font', 'typeface', 'typography', 'border', 'stroke', 'outline',
+        'shadow', 'glow', 'opacity', 'transparent', 'align', 'alignment',
+        'layout', 'columns', 'background', 'foreground', 'hover', 'accent',
+        'black', 'white', 'red', 'green', 'blue', 'yellow', 'orange', 'purple',
+        'pink', 'grey', 'gray', 'navy', 'teal', 'cyan', 'magenta', 'gold', 'beige',
+        // semantic / aesthetic asks
+        'pop', 'premium', 'modern', 'sleek', 'clean', 'professional', 'playful',
+        'elegant', 'bold', 'subtle', 'minimal', 'minimalistic', 'contrast',
+        'roomier', 'airy', 'tighter', 'softer', 'warmer', 'cooler', 'refined',
+        'polished', 'breathing', 'personality', 'aesthetic'
+    ].join('|') +
+    ')\\b', 'i');
+const ADD_PROPERTY_RE = /\b(add|create|introduce|expose|support)\b/;
+const PROPERTY_WORD_RE = /\b(property|properties|setting|settings|control|controls)\b/;
+const MAKE_EDITABLE_RE = /\bmake\s+(?:it|the\s+\w+)\s+(?:editable|customizable|configurable)\b/;
+function looksLikeNewProperty(message) {
+    const text = message.toLowerCase();
+    if (MAKE_EDITABLE_RE.test(text))
+        return true;
+    return PROPERTY_WORD_RE.test(text) && ADD_PROPERTY_RE.test(text);
+}
+function hasDesignValueSignal(message) {
+    if (/#([0-9a-f]{3}|[0-9a-f]{6})\b/i.test(message))
+        return true;
+    if (/rgba?\(/i.test(message))
+        return true;
+    if (/\d+(?:\.\d+)?\s*(px|%|rem|em)\b/i.test(message))
+        return true;
+    return DESIGN_VALUE_RE.test(message);
+}
+/**
+ * Bucket a single chat message. Pure and deterministic — the same heuristics the
+ * resolver uses for property naming, plus a few value/verb signals.
+ */
+export function classifyDesignIntent(message, schema) {
+    const raw = String(message ?? '').trim();
+    if (!raw)
+        return { bucket: 'not_design', propertyIds: [], detail: 'empty message' };
+    // 1. Explicitly asks to add a design property/setting → generation.
+    //    Checked before resolution: "add a shadow property" is about creating the
+    //    property, not about flipping the existing one.
+    if (looksLikeNewProperty(raw)) {
+        return { bucket: 'new_property', propertyIds: [], detail: 'asks for a new design property' };
+    }
+    // 2. The deterministic resolver can satisfy it outright.
+    if (resolveDesignIntent(raw, schema) !== null) {
+        return { bucket: 'literal_arithmetic', propertyIds: [], detail: 'resolved deterministically' };
+    }
+    // 3. A design property is named but the value needs judgment.
+    const named = scoreDesignProperties(raw, schema);
+    if (named.length > 0) {
+        const textNamed = named.find((entry) => TEXT_CONTROLLERS.has(entry.property.controllerType));
+        if (textNamed) {
+            return {
+                bucket: 'free_text',
+                propertyIds: [textNamed.property.id],
+                detail: 'names a text property without a quoted value'
+            };
+        }
+        return {
+            bucket: 'bounded_set_semantic',
+            propertyIds: named.map((entry) => entry.property.id),
+            detail: 'design property named, value needs judgment',
+            shape: 'property_named'
+        };
+    }
+    // 4. A design value is stated but which property it targets needs judgment.
+    if (hasDesignValueSignal(raw)) {
+        return {
+            bucket: 'bounded_set_semantic',
+            propertyIds: [],
+            detail: 'design value stated, property unresolved',
+            shape: 'value_only'
+        };
+    }
+    return { bucket: 'not_design', propertyIds: [], detail: 'no design signal' };
+}
+/**
+ * The one bucket Jev is asked about. It is the residual where a design property
+ * or value needs judgment but the value space is closed — `free_text` and
+ * `new_property` can only fall through (Jev cannot generate), and `not_design`
+ * is not a design ask at all, so neither is ever dialed.
+ */
+export const JEV_CANDIDATE_BUCKET = 'bounded_set_semantic';
+export function isJevCandidateBucket(bucket) {
+    return bucket === JEV_CANDIDATE_BUCKET;
+}
+const STORAGE_KEY = 'widgetCreator_designIntentTelemetry_v2';
+const REPORT_EVERY = 25;
+function emptyBuckets() {
+    return {
+        literal_arithmetic: 0,
+        bounded_set_semantic: 0,
+        free_text: 0,
+        new_property: 0,
+        not_design: 0
+    };
+}
+function emptyShapeBreakdown() {
+    return { attempts: 0, acted: 0, fellThrough: 0, byReason: {} };
+}
+function emptyJev() {
+    return {
+        attempts: 0,
+        acted: 0,
+        fellThrough: 0,
+        byReason: {},
+        byShape: {
+            property_named: emptyShapeBreakdown(),
+            value_only: emptyShapeBreakdown(),
+            unknown: emptyShapeBreakdown()
+        },
+        offBucketDials: 0
+    };
+}
+function readCount(value) {
+    return typeof value === 'number' && Number.isFinite(value) && value >= 0 ? value : 0;
+}
+function readReasons(value) {
+    if (!value || typeof value !== 'object')
+        return {};
+    const out = {};
+    for (const [reason, count] of Object.entries(value)) {
+        out[reason] = readCount(count);
+    }
+    return out;
+}
+function readShapeBreakdown(value) {
+    const raw = (value ?? {});
+    return {
+        attempts: readCount(raw.attempts),
+        acted: readCount(raw.acted),
+        fellThrough: readCount(raw.fellThrough),
+        byReason: readReasons(raw.byReason)
+    };
+}
+/**
+ * Parse one persisted payload. Pure, so the upgrade path — every shape recorded
+ * before `byShape` existed — is testable without a browser: counts that are
+ * absent, negative or not numbers become zero, and the flat reason map it does
+ * have is kept.
+ */
+export function parsePersistedTelemetry(raw) {
+    const base = {
+        total: 0,
+        noSchema: 0,
+        buckets: emptyBuckets(),
+        jev: emptyJev()
+    };
+    if (!raw)
+        return base;
+    let parsed;
+    try {
+        parsed = JSON.parse(raw);
+    }
+    catch {
+        return base;
+    }
+    if (!parsed || typeof parsed !== 'object')
+        return base;
+    const buckets = { ...base.buckets };
+    for (const bucket of DESIGN_INTENT_BUCKETS) {
+        buckets[bucket] = readCount((parsed.buckets ?? {})[bucket]);
+    }
+    return {
+        total: readCount(parsed.total),
+        noSchema: readCount(parsed.noSchema),
+        buckets,
+        jev: {
+            attempts: readCount(parsed.jev?.attempts),
+            acted: readCount(parsed.jev?.acted),
+            fellThrough: readCount(parsed.jev?.fellThrough),
+            byReason: readReasons(parsed.jev?.byReason),
+            byShape: {
+                property_named: readShapeBreakdown(parsed.jev?.byShape?.property_named),
+                value_only: readShapeBreakdown(parsed.jev?.byShape?.value_only),
+                unknown: readShapeBreakdown(parsed.jev?.byShape?.unknown)
+            },
+            offBucketDials: readCount(parsed.jev?.offBucketDials)
+        }
+    };
+}
+function loadPersisted() {
+    try {
+        if (typeof localStorage === 'undefined')
+            return parsePersistedTelemetry(null);
+        return parsePersistedTelemetry(localStorage.getItem(STORAGE_KEY));
+    }
+    catch {
+        return parsePersistedTelemetry(null);
+    }
+}
+let state = loadPersisted();
+const recent = [];
+const RECENT_CAP = 100;
+function persist() {
+    try {
+        if (typeof localStorage === 'undefined')
+            return;
+        localStorage.setItem(STORAGE_KEY, JSON.stringify(state));
+    }
+    catch {
+        /* a full/blocked store must never break a chat send */
+    }
+}
+/**
+ * Record one canvas-chat send and bucket it. Shadow mode: this does not change
+ * any behavior — the caller generates exactly as it did before. Returns the
+ * classification so a caller could log or assert on it.
+ */
+export function recordDesignIntentTelemetry(message, schema) {
+    const hasSchema = collectDesignProperties(schema).length > 0;
+    const classification = hasSchema
+        ? classifyDesignIntent(message, schema)
+        : { bucket: 'not_design', propertyIds: [], detail: 'no design schema loaded' };
+    state.total += 1;
+    if (!hasSchema)
+        state.noSchema += 1;
+    else
+        state.buckets[classification.bucket] += 1;
+    const event = {
+        ...classification,
+        messageLength: String(message ?? '').length,
+        at: Date.now()
+    };
+    recent.push(event);
+    if (recent.length > RECENT_CAP)
+        recent.shift();
+    persist();
+    const counted = state.total - state.noSchema;
+    console.log(`[DesignIntentTelemetry] bucket=${classification.bucket}` +
+        ` shape=${classification.shape ?? 'n/a'}` +
+        ` detail="${classification.detail}"` +
+        ` schema=${hasSchema ? 'yes' : 'no'}` +
+        ` counted=${counted}`);
+    if (counted > 0 && counted % REPORT_EVERY === 0) {
+        console.log(formatDesignIntentTelemetryReport(getDesignIntentTelemetrySnapshot()));
+    }
+    return classification;
+}
+/**
+ * Record the outcome of one Jev fallback dial. This closes the measurement
+ * loop: the bucketing tells us how big the candidate pool is, and this tells us
+ * how much of it Jev actually resolves — the number that earns the flip from
+ * `shadow` to `enforce`.
+ */
+export function recordDesignIntentJevOutcome(outcome) {
+    state.jev.attempts += 1;
+    if (outcome.acted)
+        state.jev.acted += 1;
+    else
+        state.jev.fellThrough += 1;
+    const reason = outcome.reason || 'unknown';
+    state.jev.byReason[reason] = (state.jev.byReason[reason] ?? 0) + 1;
+    // Attribute the dial to the ask shape that produced it: this is what turns a
+    // reason count into a decision about the option strings or the confidence
+    // floor, since each shape puts a different question under load.
+    const shape = outcome.classification?.shape ?? 'unknown';
+    const slice = state.jev.byShape[shape] ?? emptyShapeBreakdown();
+    slice.attempts += 1;
+    if (outcome.acted)
+        slice.acted += 1;
+    else
+        slice.fellThrough += 1;
+    slice.byReason[reason] = (slice.byReason[reason] ?? 0) + 1;
+    state.jev.byShape[shape] = slice;
+    // Only the bounded-set-semantic bucket is ever dialed by design, so a dial on
+    // any other bucket is counted as the anomaly it is instead of being mixed in.
+    const bucket = outcome.classification?.bucket;
+    if (bucket && bucket !== JEV_CANDIDATE_BUCKET)
+        state.jev.offBucketDials += 1;
+    persist();
+    console.log(`[DesignIntentTelemetry] jev source=${outcome.source} acted=${outcome.acted} reason=${reason}` +
+        ` shape=${shape}` +
+        ` resolved=${state.jev.acted}/${state.jev.attempts}`);
+}
+export function getDesignIntentTelemetrySnapshot() {
+    const counted = state.total - state.noSchema;
+    const fractions = emptyBuckets();
+    for (const bucket of DESIGN_INTENT_BUCKETS) {
+        fractions[bucket] = counted > 0 ? state.buckets[bucket] / counted : 0;
+    }
+    const modelShare = counted > 0
+        ? (state.buckets.bounded_set_semantic + state.buckets.free_text + state.buckets.new_property) / counted
+        : 0;
+    return {
+        total: state.total,
+        counted,
+        noSchema: state.noSchema,
+        buckets: { ...state.buckets },
+        fractions,
+        deterministicShare: fractions.literal_arithmetic,
+        modelShare,
+        jev: {
+            attempts: state.jev.attempts,
+            acted: state.jev.acted,
+            fellThrough: state.jev.fellThrough,
+            resolvedShare: share(state.jev.acted, state.jev.attempts),
+            byReason: { ...state.jev.byReason },
+            byShape: {
+                property_named: withShare(state.jev.byShape.property_named),
+                value_only: withShare(state.jev.byShape.value_only),
+                unknown: withShare(state.jev.byShape.unknown)
+            },
+            offBucketDials: state.jev.offBucketDials
+        }
+    };
+}
+function share(part, whole) {
+    return whole > 0 ? part / whole : 0;
+}
+function withShare(breakdown) {
+    return {
+        attempts: breakdown.attempts,
+        acted: breakdown.acted,
+        fellThrough: breakdown.fellThrough,
+        resolvedShare: share(breakdown.acted, breakdown.attempts),
+        byReason: { ...breakdown.byReason }
+    };
+}
+function pct(n) {
+    return `${(n * 100).toFixed(1)}%`;
+}
+/**
+ * What each known reason implicates, so a count can be read as a decision rather
+ * than a number. The gateway owns this vocabulary; this is the report's mirror
+ * of it, and a reason the mirror does not know renders as unrecognised — visible,
+ * never silently given a guessed knob.
+ */
+const JEV_REASON_LEVER = {
+    resolved: 'acted (the win)',
+    shadow: 'the rollout mode; enforce would act on this',
+    low_confidence: 'the confidence floor on the *property* question',
+    low_confidence_value: 'the confidence floor on the *value* question',
+    ambiguous_choice: 'the property option strings (two properties read alike)',
+    none_of_these: 'the property option strings (nothing matched the ask)',
+    unusable_property_answer: 'the question shape, not the option strings',
+    no_bounded_value: 'the roster: the chosen property has no closed value space',
+    unknown_property: 'internal: the plan and the schema disagreed',
+    no_value_stated: 'the value option strings (no bucket matched the ask)',
+    unusable_value_answer: 'the question shape, not the option strings',
+    no_color_baseline: 'not tunable: the current colour is unreadable',
+    no_numeric_baseline: 'not tunable: the current value is unreadable',
+    no_change: 'not tunable: the derived value already matched',
+    mode_off: 'the rollout mode, not the strings',
+    no_design_properties: 'the roster: the schema exposed nothing resolvable',
+    error: 'the gateway, not the questions',
+    timeout: 'the transport, not the questions',
+    circuit_open: 'the transport, not the questions',
+    no_response: 'the client could not reach the gateway'
+};
+/** Which question carried the ask, printed beside each shape's counters. */
+const JEV_SHAPE_CONTEXT = {
+    property_named: 'the value question was under load',
+    value_only: 'the property question was under load',
+    unknown: 'unattributed'
+};
+function reasonLever(reason) {
+    const lever = JEV_REASON_LEVER[reason];
+    return lever ? ` — ${lever}` : ' — unrecognised reason';
+}
+/** `reason=count` pairs, most frequent first. */
+function formatReasonCounts(byReason) {
+    return Object.entries(byReason)
+        .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
+        .map(([reason, count]) => `${reason}=${count}`)
+        .join(', ');
+}
+export function formatDesignIntentTelemetryReport(snapshot) {
+    const lines = [
+        `[DesignIntentTelemetry] report — ${snapshot.counted} classified of ${snapshot.total} sends` +
+            ` (${snapshot.noSchema} without a design schema)`,
+        `  literal/arithmetic (deterministic): ${snapshot.buckets.literal_arithmetic} (${pct(snapshot.fractions.literal_arithmetic)})`,
+        `  bounded-set semantic (JEV-shaped):  ${snapshot.buckets.bounded_set_semantic} (${pct(snapshot.fractions.bounded_set_semantic)})`,
+        `  free text (LLM):                    ${snapshot.buckets.free_text} (${pct(snapshot.fractions.free_text)})`,
+        `  new property (LLM):                 ${snapshot.buckets.new_property} (${pct(snapshot.fractions.new_property)})`,
+        `  not a design ask (codegen):         ${snapshot.buckets.not_design} (${pct(snapshot.fractions.not_design)})`,
+        `  → could skip the LLM (deterministic share): ${pct(snapshot.deterministicShare)}`,
+        `  → design asks needing a model:              ${pct(snapshot.modelShare)}`
+    ];
+    const { jev } = snapshot;
+    lines.push(`  Jev fallback on the bounded-set-semantic bucket: ${jev.acted}/${jev.attempts} resolved` +
+        ` (${pct(jev.resolvedShare)}), ${jev.fellThrough} fell through`);
+    if (jev.attempts > 0) {
+        // The overall count comes first, then the same counts split by the ask
+        // shape that produced them — because "low_confidence=4" means a different
+        // thing when the property was named than when it was not.
+        lines.push(`    reasons overall: ${formatReasonCounts(jev.byReason)}`);
+        lines.push('    by ask shape (which question carried the ask):');
+        for (const shape of JE_ASK_SHAPES) {
+            const slice = jev.byShape[shape];
+            if (slice.attempts === 0)
+                continue;
+            lines.push(`      ${shape} — ${JEV_SHAPE_CONTEXT[shape]}: ${slice.acted}/${slice.attempts} resolved` +
+                ` (${pct(slice.resolvedShare)}), ${slice.fellThrough} fell through`);
+            for (const [reason, count] of Object.entries(slice.byReason).sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))) {
+                lines.push(`        ${reason}=${count}${reasonLever(reason)}`);
+            }
+        }
+        if (jev.offBucketDials > 0) {
+            lines.push(`    ⚠ ${jev.offBucketDials} dials were attributed to another bucket —` +
+                ` only ${JEV_CANDIDATE_BUCKET} is ever dialed by design.`);
+        }
+    }
+    return lines.join('\n');
+}
+/** Recent events (last 100, newest last) — for a quick look in the console. */
+export function getRecentDesignIntentEvents() {
+    return [...recent];
+}
+export function resetDesignIntentTelemetry() {
+    state = { total: 0, noSchema: 0, buckets: emptyBuckets(), jev: emptyJev() };
+    recent.length = 0;
+    persist();
+}
+// Dev/ops handle: inspect or reset from the console without a rebuild.
+if (typeof window !== 'undefined') {
+    window.__widgeticDesignIntentTelemetry = {
+        snapshot: getDesignIntentTelemetrySnapshot,
+        report: () => {
+            const snapshot = getDesignIntentTelemetrySnapshot();
+            console.log(formatDesignIntentTelemetryReport(snapshot));
+            return snapshot;
+        },
+        recent: getRecentDesignIntentEvents,
+        reset: resetDesignIntentTelemetry
+    };
+}

package/dist/utils/designIntentTelemetry.test.d.ts

@@ -0,0 +1 @@
+export {};