@hachej/boring-feedback 0.0.0-stage → 0.2.5

package/dist/page/voice/capture.js

@@ -0,0 +1,186 @@
+// Voice capture for a feedback session: the person's microphone recorded while they point, with the page-clock time of the first
+// recorded sample so speech can be aligned with the pointer (./align.ts).
+//
+// Microphone access (getUserMedia) exists only in secure contexts (https or localhost) and needs the person's permission. The
+// capture never opens it on load: only `start()`, called from the person's click, does, through an injectable `getUserMedia`
+// whose default feature-detects `navigator.mediaDevices.getUserMedia` and refuses with a reason when it is missing. A host that
+// owns microphone policy passes its own. `npm run check` (scripts/check-platform-apis.mjs) allowlists this file on that condition.
+//
+// The clock. MediaRecorder hands audio over late and in chunks, so neither its `start` event nor a chunk's arrival says when the
+// first recorded sample was heard. `npm run feedback:voice-sync` (tools/feedback-voice-sync.mjs) measured seven candidate
+// strategies in Chromium against an independent reader of the same microphone; the page time read immediately before
+// `MediaRecorder.start()` was the most accurate (2–4 ms mean, under 9 ms max over three batches, with the fake device's dropped
+// time removed), while the `start` event is about 60 ms late (up to 125 ms) and stop-minus-duration 30–40 ms late. So
+// `startedAt` is that time.
+// `endedAt` is `startedAt` plus the decoded duration when the page can decode the recording, else the `stop` event's time.
+// Measured in Chromium only; a microphone that warms up after getUserMedia resolves (no audio yet when recording starts) would
+// make `startedAt` early by the warm-up.
+export const DEFAULT_MAX_VOICE_MS = 5 * 60_000;
+/** Container types tried in order; the first the browser records wins. */
+export const VOICE_MIME_TYPES = ['audio/webm;codecs=opus', 'audio/webm', 'audio/mp4', 'audio/ogg;codecs=opus'];
+/** The browser's microphone, or null where the page cannot have one (insecure origin, no media devices). */
+export function browserGetUserMedia() {
+    if (typeof globalThis.navigator?.mediaDevices?.getUserMedia !== 'function')
+        return null;
+    const devices = globalThis.navigator.mediaDevices;
+    return constraints => devices.getUserMedia(constraints);
+}
+/** Short human messages, one per refusal; each says what to do next. */
+export const VOICE_REFUSALS = {
+    denied: 'Microphone blocked. Allow it for this site in the browser, then press Record again.',
+    'no-microphone': 'No microphone found. Connect one, then press Record again.',
+    busy: 'The microphone is busy or could not be opened. Close other apps using it and try again.',
+    insecure: 'Recording needs a secure page (https or localhost).',
+    unsupported: 'This browser cannot record audio here.',
+    failed: 'The microphone could not be started.',
+};
+const refused = (code, detail) => ({ kind: 'refused', code, reason: detail ? `${VOICE_REFUSALS[code]} (${detail})` : VOICE_REFUSALS[code] });
+function refusalOf(error) {
+    const name = error?.name;
+    if (name === 'NotAllowedError' || name === 'PermissionDeniedError' || name === 'SecurityError')
+        return refused('denied');
+    if (name === 'NotFoundError' || name === 'DevicesNotFoundError' || name === 'OverconstrainedError')
+        return refused('no-microphone');
+    if (name === 'NotReadableError' || name === 'TrackStartError' || name === 'AbortError')
+        return refused('busy');
+    return refused('failed', error instanceof Error ? error.message : String(error));
+}
+/** Media devices missing on a plain-http page: the usual reason is an insecure origin (http on a name that is not local; a local
+ * http page has them). Wording only; the capability itself is feature-detected (`browserGetUserMedia`). */
+const plainHttpPage = () => globalThis.location?.protocol === 'http:';
+const release = (stream) => { for (const track of stream?.getTracks() ?? [])
+    track.stop(); };
+/** Duration in milliseconds of recorded audio, when the page can decode it (OfflineAudioContext or AudioContext). */
+async function decodedMs(audio) {
+    try {
+        const Offline = globalThis.OfflineAudioContext;
+        if (typeof Offline !== 'function' || audio.size === 0)
+            return null;
+        const context = new Offline(1, 1, 48_000);
+        const buffer = await context.decodeAudioData(await audio.arrayBuffer());
+        return Number.isFinite(buffer.duration) ? buffer.duration * 1000 : null;
+    }
+    catch {
+        return null;
+    }
+}
+export function createVoiceCapture(options = {}) {
+    const now = options.now ?? (() => performance.now());
+    const maxMs = options.maxMs ?? DEFAULT_MAX_VOICE_MS;
+    let session = null;
+    let opening = false;
+    let cancelledWhileOpening = false;
+    const finish = (current, truncated) => {
+        if (current.timer !== null) {
+            clearTimeout(current.timer);
+            current.timer = null;
+        }
+        if (truncated)
+            current.truncated = true;
+        if (current.recorder.state !== 'inactive') {
+            try {
+                current.recorder.stop();
+            }
+            catch { /* already stopping */ }
+        }
+    };
+    return {
+        async start() {
+            if (session || opening)
+                return { kind: 'refused', code: 'failed', reason: 'Already recording.' };
+            const Recorder = globalThis.MediaRecorder;
+            const open = options.getUserMedia ?? browserGetUserMedia();
+            // No media devices on an insecure origin: say that, not "no microphone" (the person can fix the address, not the hardware).
+            if (!open)
+                return refused(plainHttpPage() ? 'insecure' : 'unsupported');
+            if (typeof Recorder !== 'function')
+                return refused('unsupported', 'no MediaRecorder');
+            // getUserMedia is called synchronously from here, inside the person's click (no await before it), so browsers that tie the
+            // permission prompt to a user gesture show it.
+            opening = true;
+            cancelledWhileOpening = false;
+            let stream;
+            try {
+                stream = await open({ audio: true });
+            }
+            catch (error) {
+                opening = false;
+                return refusalOf(error);
+            }
+            if (cancelledWhileOpening) {
+                opening = false;
+                release(stream);
+                return { kind: 'refused', code: 'failed', reason: 'Cancelled before the microphone opened.' };
+            }
+            const supported = (type) => { try {
+                return typeof Recorder.isTypeSupported !== 'function' || Recorder.isTypeSupported(type);
+            }
+            catch {
+                return false;
+            } };
+            const preferred = VOICE_MIME_TYPES.find(supported);
+            let recorder;
+            try {
+                recorder = preferred ? new Recorder(stream, { mimeType: preferred }) : new Recorder(stream);
+            }
+            catch (error) {
+                opening = false;
+                release(stream);
+                return refused('unsupported', error instanceof Error ? error.message : String(error));
+            }
+            let settle;
+            const current = {
+                stream, recorder, mimeType: preferred ?? '', chunks: [], startedAt: null, stoppedAt: null, truncated: false, timer: null,
+                stopped: new Promise(resolve => { settle = resolve; }), cancelled: false,
+            };
+            recorder.addEventListener('dataavailable', event => { if (event.data && event.data.size > 0)
+                current.chunks.push(event.data); });
+            recorder.addEventListener('stop', () => { current.stoppedAt = now(); release(stream); settle(); });
+            recorder.addEventListener('error', () => { current.stoppedAt ??= now(); release(stream); settle(); });
+            try {
+                current.startedAt = now();
+                recorder.start(1000);
+            }
+            catch (error) {
+                opening = false;
+                release(stream);
+                return refused('failed', error instanceof Error ? error.message : String(error));
+            }
+            if (Number.isFinite(maxMs) && maxMs > 0)
+                current.timer = setTimeout(() => finish(current, true), maxMs);
+            session = current;
+            opening = false;
+            return { kind: 'recording' };
+        },
+        async stop() {
+            const current = session;
+            if (!current)
+                return null;
+            finish(current, false);
+            await current.stopped;
+            session = null;
+            if (current.cancelled)
+                return null;
+            const mimeType = current.recorder.mimeType || current.mimeType || current.chunks[0]?.type || 'audio/webm';
+            const audio = new Blob(current.chunks, { type: mimeType });
+            if (audio.size === 0)
+                return null;
+            const startedAt = current.startedAt ?? current.stoppedAt ?? now();
+            const duration = await decodedMs(audio);
+            const endedAt = duration === null ? Math.max(startedAt, current.stoppedAt ?? now()) : startedAt + duration;
+            return { audio, mimeType, startedAt, endedAt, truncated: current.truncated };
+        },
+        cancel() {
+            const current = session;
+            if (opening)
+                cancelledWhileOpening = true;
+            if (!current)
+                return;
+            session = null;
+            current.cancelled = true;
+            current.chunks.length = 0;
+            finish(current, false);
+            release(current.stream);
+        },
+    };
+}

package/dist/page/voice/index.d.ts

@@ -0,0 +1,3 @@
+export { createVoiceCapture, browserGetUserMedia, DEFAULT_MAX_VOICE_MS, VOICE_MIME_TYPES, VOICE_REFUSALS, type VoiceCapture, type VoiceRefusal, type VoiceCaptureOptions, type VoiceStart, type VoiceRecording, } from './capture.js';
+export { alignSpeech, PIN_AFTER_MS, type SpeechSegment, type SpeechWord, type PointerSample, type PinMark, type AlignedSpeech } from './align.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/page/voice/index.js

@@ -0,0 +1,3 @@
+// Voice in a feedback session: the microphone capture and the alignment of what was said with where the pointer was.
+export { createVoiceCapture, browserGetUserMedia, DEFAULT_MAX_VOICE_MS, VOICE_MIME_TYPES, VOICE_REFUSALS, } from './capture.js';
+export { alignSpeech, PIN_AFTER_MS } from './align.js';

package/dist/preview/agent.d.ts

@@ -0,0 +1,34 @@
+import type { Api, Message, Model, Models, Tool } from '@earendil-works/pi-ai';
+import type { PreviewPage } from './page.js';
+import { type PreviewAnswer } from '../format/index.js';
+export declare const PREVIEW_SYSTEM_PROMPT = "You preview a requested change on a live web page, for a person who will approve or discard it. Nothing you do is saved.\nUse only the tools. Reference elements by the pinned ids (p1, p2...) or by ids from inspect (e1, e2...). Call inspect when you need to see the page.\nPrefer the smallest change that does what was asked, on the pinned elements. set_style takes plain CSS declarations (\"background-color: #2f9e44; color: #ffffff\").\nText shown as masked is private: never guess or rewrite it. When the change is done, answer with one short sentence saying what you changed.";
+/** The page tools, as Pi tool declarations. */
+export declare const PREVIEW_TOOLS: readonly Tool[];
+export interface PreviewAgentOptions {
+    readonly models: Models;
+    readonly model: Model<Api>;
+    readonly page: PreviewPage;
+    /** What to preview, from the server's `browser_preview` call. */
+    readonly instructions: string;
+    /** Model turns per `say`, tool calls included. Default 8. */
+    readonly maxSteps?: number;
+}
+export interface PreviewAgent {
+    /** Runs the loop on the first instructions (`say()` with no text) or on a follow-up ("darker"). Resolves with the final sentence. */
+    readonly say: (text?: string, signal?: AbortSignal) => Promise<{
+        readonly text: string;
+    } | {
+        readonly error: string;
+    }>;
+    /** Everything sent to and received from the model, in order. */
+    readonly messages: () => readonly Message[];
+    /** The approved answer from the current page changes and the last sentence. */
+    readonly approve: () => PreviewAnswer;
+    /** Reverts the page and returns the discarded answer. */
+    readonly discard: () => PreviewAnswer;
+}
+/** The first message: the instructions and the pinned elements as the privacy policy names them. */
+export declare function previewBrief(instructions: string, page: PreviewPage): string;
+/** A preview subagent over one page. Several `say` calls continue the same conversation; they must not overlap. */
+export declare function createPreviewAgent(options: PreviewAgentOptions): PreviewAgent;
+//# sourceMappingURL=agent.d.ts.map

package/dist/preview/agent.js

@@ -0,0 +1,91 @@
+// The browser preview subagent (FEEDBACK.md, "Preview"): a small tool-calling loop on Pi's model layer, in the page. The host gives
+// it native Pi `Models` whose provider is its model gateway (`gatewayProvider` from `@hachej/boring-agent/gateway-provider`), so the page
+// holds no key; this module never builds a provider itself. Tool calls are validated by Pi (`validateToolCall`) and executed by the
+// preview page tools (./page.ts) only: there is no other tool, so a call to anything else is an error result. The loop is in memory
+// on purpose: a preview is not saved, and a reload reverts the page anyway; the durable part is the server's `browser_preview` wait.
+import { Type, validateToolCall } from '@earendil-works/pi-ai';
+import { PREVIEW_LIMITS } from '../format/index.js';
+export const PREVIEW_SYSTEM_PROMPT = `You preview a requested change on a live web page, for a person who will approve or discard it. Nothing you do is saved.
+Use only the tools. Reference elements by the pinned ids (p1, p2...) or by ids from inspect (e1, e2...). Call inspect when you need to see the page.
+Prefer the smallest change that does what was asked, on the pinned elements. set_style takes plain CSS declarations ("background-color: #2f9e44; color: #ffffff").
+Text shown as masked is private: never guess or rewrite it. When the change is done, answer with one short sentence saying what you changed.`;
+const element = Type.String({ minLength: 1, maxLength: 20, description: 'A pin id (p1...) or an id from inspect (e1...).' });
+/** The page tools, as Pi tool declarations. */
+export const PREVIEW_TOOLS = Object.freeze([
+    { name: 'inspect', description: 'The masked outline of the application page with element ids, the pinned elements first, and a few computed styles.', parameters: Type.Object({}, { additionalProperties: false }) },
+    { name: 'set_style', description: 'Set inline CSS properties on one element (colors, borders, spacing, fonts, sizes, display). No url(), no !important.', parameters: Type.Object({ element, css: Type.String({ minLength: 1, maxLength: 600 }) }, { additionalProperties: false }) },
+    { name: 'set_text', description: 'Replace the visible text of one text-only element (a button label, a heading...).', parameters: Type.Object({ element, text: Type.String({ minLength: 1, maxLength: 200 }) }, { additionalProperties: false }) },
+    { name: 'hide', description: 'Hide one element.', parameters: Type.Object({ element }, { additionalProperties: false }) },
+    { name: 'show', description: 'Show again an element this preview hid.', parameters: Type.Object({ element }, { additionalProperties: false }) },
+]);
+const textOf = (message) => message.content.filter(part => part.type === 'text').map(part => part.text).join('').trim();
+function run(page, call) {
+    const args = validateToolCall([...PREVIEW_TOOLS], call);
+    switch (call.name) {
+        case 'inspect': return { ok: true, message: page.inspect() };
+        case 'set_style': return page.setStyle(args['element'] ?? '', args['css'] ?? '');
+        case 'set_text': return page.setText(args['element'] ?? '', args['text'] ?? '');
+        case 'hide': return page.hide(args['element'] ?? '');
+        case 'show': return page.show(args['element'] ?? '');
+        default: return { ok: false, reason: `unknown tool ${call.name}` };
+    }
+}
+/** The first message: the instructions and the pinned elements as the privacy policy names them. */
+export function previewBrief(instructions, page) {
+    const pins = page.pins.map(pin => `- ${pin.id}: ${pin.fallback}${pin.source ? ` [source ${pin.source}]` : ''}${pin.unplaced ? ` (not usable: ${pin.unplaced})` : ''}`);
+    return `Preview this change: ${instructions}\nPinned elements:\n${pins.join('\n') || '- none: call inspect to find the element'}`;
+}
+/** A preview subagent over one page. Several `say` calls continue the same conversation; they must not overlap. */
+export function createPreviewAgent(options) {
+    const { models, model, page } = options;
+    const maxSteps = options.maxSteps ?? 8;
+    const messages = [];
+    let summary = '';
+    let running = false;
+    const say = async (text, signal) => {
+        if (running)
+            return { error: 'The preview is still working.' };
+        running = true;
+        try {
+            const content = messages.length === 0 ? previewBrief(options.instructions, page) + (text ? `\n${text}` : '') : text?.trim();
+            if (!content)
+                return { error: 'Say what to change.' };
+            messages.push({ role: 'user', content, timestamp: Date.now() });
+            for (let step = 0; step < maxSteps; step++) {
+                if (signal?.aborted)
+                    return { error: 'Stopped.' };
+                const reply = await models.complete(model, { systemPrompt: PREVIEW_SYSTEM_PROMPT, messages: [...messages], tools: [...PREVIEW_TOOLS] }, signal ? { signal } : {});
+                messages.push(reply);
+                if (reply.stopReason === 'error' || reply.stopReason === 'aborted')
+                    return { error: reply.errorMessage ?? 'The model could not answer.' };
+                const calls = reply.content.filter((part) => part.type === 'toolCall');
+                if (!calls.length) {
+                    summary = textOf(reply).slice(0, PREVIEW_LIMITS.summary);
+                    return { text: summary };
+                }
+                for (const call of calls) {
+                    let outcome;
+                    try {
+                        outcome = run(page, call);
+                    }
+                    catch (error) {
+                        outcome = { ok: false, reason: error instanceof Error ? error.message : 'invalid call' };
+                    }
+                    const result = { role: 'toolResult', toolCallId: call.id, toolName: call.name, isError: !outcome.ok,
+                        content: [{ type: 'text', text: outcome.ok ? outcome.message : `Refused: ${outcome.reason}` }], timestamp: Date.now() };
+                    messages.push(result);
+                }
+            }
+            return { error: `The preview stopped after ${maxSteps} steps.` };
+        }
+        finally {
+            running = false;
+        }
+    };
+    return Object.freeze({
+        say,
+        messages: () => [...messages],
+        approve: () => ({ kind: 'approved', summary: summary || 'Previewed on the page.', changes: page.changes().slice(0, PREVIEW_LIMITS.changes) }),
+        discard: () => { page.revert(); return { kind: 'discarded' }; },
+    });
+}

package/dist/preview/index.d.ts

@@ -0,0 +1,5 @@
+export { createPreviewPage, parseDeclarations, PREVIEW_STYLE_PROPERTIES, type PreviewPage, type PreviewPageOptions, type PreviewPin, type PreviewOutcome } from './page.js';
+export { createPreviewAgent, previewBrief, PREVIEW_SYSTEM_PROMPT, PREVIEW_TOOLS, type PreviewAgent, type PreviewAgentOptions } from './agent.js';
+export { createPreviewSession, pendingBrowserTasks, BROWSER_PREVIEW_TOOL, type PreviewSession, type PreviewSessionOptions, type PreviewSnapshot, type PreviewStatus, type PendingBrowserTask, type PreviewAnswerOutcome, } from './session.js';
+export type { PreviewAnswer, PreviewChange } from '../format/index.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/preview/index.js

@@ -0,0 +1,6 @@
+// @hachej/boring-feedback/preview (FEEDBACK.md, "Preview"): the browser side of `browser_preview`. The page tools over the live
+// application root (privacy policy in, logged and revertible changes out), the in-page Pi subagent that drives them through the
+// host's model gateway, and the session the preview banner shows. Imports `./page`, `./format` and Pi's model layer only.
+export { createPreviewPage, parseDeclarations, PREVIEW_STYLE_PROPERTIES } from './page.js';
+export { createPreviewAgent, previewBrief, PREVIEW_SYSTEM_PROMPT, PREVIEW_TOOLS } from './agent.js';
+export { createPreviewSession, pendingBrowserTasks, BROWSER_PREVIEW_TOOL, } from './session.js';

package/dist/preview/page.d.ts

@@ -0,0 +1,50 @@
+import { type PrivacyPolicy } from '../page/index.js';
+import type { PreviewChange } from '../format/index.js';
+/** CSS properties a preview may set inline. Layout-breaking or resource-loading ones (`content`, `cursor`, `list-style-image`...) are not here. */
+export declare const PREVIEW_STYLE_PROPERTIES: readonly string[];
+export type PreviewOutcome = {
+    readonly ok: true;
+    readonly message: string;
+} | {
+    readonly ok: false;
+    readonly reason: string;
+};
+export interface PreviewPin {
+    /** `p1`, `p2`... in the report's anchor order. */
+    readonly id: string;
+    readonly fallback: string;
+    readonly source?: string;
+    /** Why the pin is not on this page now (missing, ambiguous...); the pin cannot be changed then. */
+    readonly unplaced?: string;
+}
+export interface PreviewPage {
+    readonly pins: readonly PreviewPin[];
+    /** The masked outline of the application root with element ids, pins first. */
+    readonly inspect: () => string;
+    readonly setStyle: (element: string, css: string) => PreviewOutcome;
+    readonly setText: (element: string, text: string) => PreviewOutcome;
+    readonly hide: (element: string) => PreviewOutcome;
+    readonly show: (element: string) => PreviewOutcome;
+    /** Net changes so far (first `from`, last `to`; no-ops dropped), in the order they were first made. */
+    readonly changes: () => readonly PreviewChange[];
+    /** Restores every touched element's original `style` attribute and text nodes, newest first. Idempotent. */
+    readonly revert: () => void;
+}
+export interface PreviewPageOptions {
+    /** The application root. Nothing outside it can be referenced. */
+    readonly root: Element;
+    readonly policy: PrivacyPolicy;
+    /** The feedback report's anchors; `app.element@1` ones placed `exact` or `moved` become pins. */
+    readonly anchors?: readonly unknown[];
+}
+/** `prop: value; prop: value`, each checked. Returns the declarations or the reason for refusing. */
+export declare function parseDeclarations(css: string): {
+    readonly ok: true;
+    readonly declarations: readonly (readonly [string, string])[];
+} | {
+    readonly ok: false;
+    readonly reason: string;
+};
+/** Builds the preview tools over the live application root. Pins are placed now, against a fresh masked snapshot. */
+export declare function createPreviewPage(options: PreviewPageOptions): Promise<PreviewPage>;
+//# sourceMappingURL=page.d.ts.map