@agent-native/core 0.79.27 → 0.80.0

package/corpus/core/src/templates/workspace-core/.agents/skills/voice-transcription/SKILL.md

@@ -86,6 +86,7 @@ Default behavior:
 | `packages/core/src/client/transcription/use-live-transcription.ts`    | Web Speech live-transcription hook for recordings   |
 | `packages/core/src/server/transcribe-voice.ts`                        | Route handler (routes to Builder/Gemini/Groq/Whisper) |
 | `packages/core/src/transcription/builder-transcription.ts`            | Builder proxy transcription client                  |
+| `packages/core/src/voice/`                                            | Shared voice context pack, prompt, and replacement helpers |
 | `packages/core/src/secrets/register-framework-secrets.ts`             | Framework-level provider key registration           |
 
 ## Key resolution (server)
@@ -114,6 +115,15 @@ provider. Gemini uses them in the transcription prompt, Builder receives them
 as transcription/cleanup instructions, and Whisper-compatible providers receive
 them as provider prompt/context.
 
+Requests may also include a multipart `voiceContext` JSON field. Parse it with
+`parseVoiceContextPack()` from `@agent-native/core/voice` and pass the resulting
+context through `buildVoiceGuidanceBlock()`; do not hand-roll separate prompt
+formats. The context pack is for bounded snippets, active references, route/page
+metadata, and preferred vocabulary/replacements only. Browser-native and Google
+realtime final text can POST `{ text, provider, instructions, voiceContext }` to
+`/_agent-native/transcribe-voice` when AI cleanup is enabled; batch audio can
+send the same `voiceContext` field with its audio upload.
+
 Never hardcode a shared key. Never log the value. Never echo it back to the
 client.
 

package/corpus/core/src/vite/client.ts

@@ -461,6 +461,7 @@ const CORE_CLIENT_SUBPATHS = [
   "@agent-native/core/client/notifications",
   "@agent-native/core/client/progress",
   "@agent-native/core/client/transcription/use-live-transcription",
+  "@agent-native/core/voice",
 ];
 
 function getDefaultOptimizeDeps(cwd: string): string[] {
@@ -769,6 +770,7 @@ function getCoreSourceAliases(
       coreSrc,
       "client/transcription/use-live-transcription.ts",
     ),
+    "@agent-native/core/voice": path.join(coreSrc, "voice/index.ts"),
     "@agent-native/core/db": path.join(coreSrc, "db/index.ts"),
     "@agent-native/core/db/schema": path.join(coreSrc, "db/schema.ts"),
     "@agent-native/core/shared": path.join(coreSrc, "shared/index.ts"),

package/corpus/core/src/voice/index.ts

@@ -0,0 +1,18 @@
+export type {
+  VoiceContextMode,
+  VoiceContextPack,
+  VoiceContextSnippet,
+  VoiceTerm,
+} from "./voice-context.js";
+export {
+  formatVoiceContextPackForPrompt,
+  parseVoiceContextPack,
+  sanitizeVoiceContextPack,
+  voiceContextHasContent,
+  voiceContextTermsOnly,
+} from "./voice-context.js";
+export { buildVoiceGuidanceBlock } from "./voice-cleanup-prompt.js";
+export {
+  applyVoiceContextReplacements,
+  applyVoiceTermReplacements,
+} from "./voice-replacements.js";

package/corpus/core/src/voice/voice-cleanup-prompt.ts

@@ -0,0 +1,38 @@
+import {
+  formatVoiceContextPackForPrompt,
+  type VoiceContextPack,
+} from "./voice-context.js";
+
+const MAX_CUSTOM_INSTRUCTIONS_CHARS = 4_000;
+
+function cleanInstructions(value: string | undefined): string | undefined {
+  const trimmed = value?.replace(/\0/g, "").trim();
+  if (!trimmed) return undefined;
+  return trimmed.slice(0, MAX_CUSTOM_INSTRUCTIONS_CHARS);
+}
+
+export function buildVoiceGuidanceBlock({
+  instructions,
+  contextPack,
+}: {
+  instructions?: string;
+  contextPack?: VoiceContextPack | null;
+}): string | undefined {
+  const customInstructions = cleanInstructions(instructions);
+  const formattedContext = formatVoiceContextPackForPrompt(contextPack);
+  if (!customInstructions && !formattedContext) return undefined;
+
+  const sections = [
+    "Use the following voice guidance only to improve transcription, cleanup, terminology, casing, punctuation, and formatting.",
+    "Never add facts or content that are not present in the audio or transcript.",
+  ];
+
+  if (customInstructions) {
+    sections.push(`User instructions:\n${customInstructions}`);
+  }
+  if (formattedContext) {
+    sections.push(`Voice context:\n${formattedContext}`);
+  }
+
+  return sections.join("\n\n");
+}

package/corpus/core/src/voice/voice-context.ts

@@ -0,0 +1,217 @@
+export type VoiceContextMode =
+  | "dictation"
+  | "cleanup"
+  | "transcript"
+  | "title"
+  | "summary"
+  | "meeting";
+
+export interface VoiceContextSnippet {
+  label: string;
+  value: string;
+}
+
+export interface VoiceTerm {
+  term: string;
+  replacement?: string;
+  confidence?: number;
+  source?: string;
+  scope?: string;
+}
+
+export interface VoiceContextPack {
+  surface?: string;
+  mode?: VoiceContextMode | string;
+  snippets?: VoiceContextSnippet[];
+  terms?: VoiceTerm[];
+  metadata?: Record<string, string | number | boolean | null | undefined>;
+}
+
+const MAX_CONTEXT_PACK_CHARS = 64_000;
+const MAX_SNIPPETS = 8;
+const MAX_SNIPPET_VALUE_CHARS = 1_800;
+const MAX_TERMS = 80;
+const MAX_TERM_CHARS = 120;
+const MAX_METADATA_KEYS = 12;
+const MAX_METADATA_VALUE_CHARS = 160;
+
+function cleanString(value: unknown, maxChars: number): string | undefined {
+  if (typeof value !== "string") return undefined;
+  const trimmed = value.replace(/\0/g, "").trim();
+  if (!trimmed) return undefined;
+  return trimmed.slice(0, maxChars);
+}
+
+function cleanNumber(value: unknown): number | undefined {
+  if (typeof value !== "number" || !Number.isFinite(value)) return undefined;
+  return Math.min(1, Math.max(0, value));
+}
+
+export function sanitizeVoiceContextPack(
+  input: unknown,
+): VoiceContextPack | undefined {
+  if (!input || typeof input !== "object" || Array.isArray(input)) {
+    return undefined;
+  }
+
+  const raw = input as VoiceContextPack;
+  const surface = cleanString(raw.surface, 80);
+  const mode = cleanString(raw.mode, 40);
+
+  const snippets = Array.isArray(raw.snippets)
+    ? raw.snippets
+        .slice(0, MAX_SNIPPETS)
+        .map((snippet) => {
+          if (!snippet || typeof snippet !== "object") return null;
+          const candidate = snippet as Partial<VoiceContextSnippet>;
+          const label = cleanString(candidate.label, 80);
+          const value = cleanString(candidate.value, MAX_SNIPPET_VALUE_CHARS);
+          return label && value ? { label, value } : null;
+        })
+        .filter((snippet): snippet is VoiceContextSnippet => snippet !== null)
+    : undefined;
+
+  const terms = Array.isArray(raw.terms)
+    ? raw.terms
+        .slice(0, MAX_TERMS)
+        .map((term) => {
+          if (!term || typeof term !== "object") return null;
+          const candidate = term as Partial<VoiceTerm>;
+          const rawTerm = cleanString(candidate.term, MAX_TERM_CHARS);
+          if (!rawTerm || rawTerm.length < 2) return null;
+          const replacement = cleanString(
+            candidate.replacement,
+            MAX_TERM_CHARS,
+          );
+          const source = cleanString(candidate.source, 40);
+          const scope = cleanString(candidate.scope, 40);
+          const confidence = cleanNumber(candidate.confidence);
+          return {
+            term: rawTerm,
+            ...(replacement ? { replacement } : {}),
+            ...(confidence !== undefined ? { confidence } : {}),
+            ...(source ? { source } : {}),
+            ...(scope ? { scope } : {}),
+          };
+        })
+        .filter((term): term is VoiceTerm => term !== null)
+    : undefined;
+
+  const metadata =
+    raw.metadata &&
+    typeof raw.metadata === "object" &&
+    !Array.isArray(raw.metadata)
+      ? Object.fromEntries(
+          Object.entries(raw.metadata)
+            .slice(0, MAX_METADATA_KEYS)
+            .flatMap(([key, value]) => {
+              const cleanKey = cleanString(key, 60);
+              if (!cleanKey) return [];
+              if (
+                typeof value === "string" ||
+                typeof value === "number" ||
+                typeof value === "boolean" ||
+                value === null
+              ) {
+                const cleanValue =
+                  typeof value === "string"
+                    ? cleanString(value, MAX_METADATA_VALUE_CHARS)
+                    : value;
+                return cleanValue === undefined ? [] : [[cleanKey, cleanValue]];
+              }
+              return [];
+            }),
+        )
+      : undefined;
+
+  const pack: VoiceContextPack = {
+    ...(surface ? { surface } : {}),
+    ...(mode ? { mode } : {}),
+    ...(snippets?.length ? { snippets } : {}),
+    ...(terms?.length ? { terms } : {}),
+    ...(metadata && Object.keys(metadata).length > 0 ? { metadata } : {}),
+  };
+
+  return voiceContextHasContent(pack) ? pack : undefined;
+}
+
+export function parseVoiceContextPack(
+  value: string,
+): VoiceContextPack | undefined {
+  const trimmed = value.replace(/\0/g, "").trim();
+  if (!trimmed || trimmed.length > MAX_CONTEXT_PACK_CHARS) return undefined;
+  try {
+    return sanitizeVoiceContextPack(JSON.parse(trimmed));
+  } catch {
+    return undefined;
+  }
+}
+
+export function voiceContextHasContent(
+  pack: VoiceContextPack | undefined,
+): boolean {
+  return Boolean(
+    pack?.surface ||
+    pack?.mode ||
+    pack?.snippets?.length ||
+    pack?.terms?.length ||
+    (pack?.metadata && Object.keys(pack.metadata).length > 0),
+  );
+}
+
+export function voiceContextTermsOnly(
+  input: unknown,
+): VoiceContextPack | undefined {
+  const pack = sanitizeVoiceContextPack(input);
+  if (!pack?.terms?.length) return undefined;
+  return { terms: pack.terms };
+}
+
+export function formatVoiceContextPackForPrompt(input: unknown): string {
+  const pack = sanitizeVoiceContextPack(input);
+  if (!pack) return "";
+
+  const sections: string[] = [];
+  if (pack.surface) sections.push(`Surface: ${pack.surface}`);
+  if (pack.mode) sections.push(`Mode: ${pack.mode}`);
+
+  if (pack.snippets?.length) {
+    sections.push(
+      [
+        "Context snippets:",
+        ...pack.snippets.map(
+          (snippet) => `- ${snippet.label}: ${snippet.value}`,
+        ),
+      ].join("\n"),
+    );
+  }
+
+  if (pack.terms?.length) {
+    sections.push(
+      [
+        "Preferred vocabulary and casing:",
+        ...pack.terms.map((term) => {
+          const replacement =
+            term.replacement && term.replacement !== term.term
+              ? ` -> ${term.replacement}`
+              : "";
+          const source = term.source ? ` (${term.source})` : "";
+          return `- ${term.term}${replacement}${source}`;
+        }),
+      ].join("\n"),
+    );
+  }
+
+  if (pack.metadata && Object.keys(pack.metadata).length > 0) {
+    sections.push(
+      [
+        "Metadata:",
+        ...Object.entries(pack.metadata).map(([key, value]) => {
+          return `- ${key}: ${String(value)}`;
+        }),
+      ].join("\n"),
+    );
+  }
+
+  return sections.join("\n\n");
+}

package/corpus/core/src/voice/voice-replacements.ts

@@ -0,0 +1,75 @@
+import {
+  sanitizeVoiceContextPack,
+  type VoiceContextPack,
+  type VoiceTerm,
+} from "./voice-context.js";
+
+function escapeRegExp(value: string): string {
+  return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
+function isAlphaNumeric(value: string | undefined): boolean {
+  return value != null && /[\p{L}\p{N}_]/u.test(value);
+}
+
+function isSafeVoiceReplacementBoundary(
+  text: string,
+  index: number,
+  direction: "before" | "after",
+): boolean {
+  const char = text[index];
+  if (char == null) return true;
+  if (isAlphaNumeric(char)) return false;
+  if (char === "@" || char === "/" || char === "\\" || char === ":") {
+    return false;
+  }
+  if (char === "." || char === "-" || char === "+") {
+    const neighbor = direction === "before" ? text[index - 1] : text[index + 1];
+    return !isAlphaNumeric(neighbor);
+  }
+  return true;
+}
+
+export function applyVoiceTermReplacements(
+  text: string,
+  terms: readonly VoiceTerm[] | undefined,
+): string {
+  if (!text || !terms?.length) return text;
+
+  let next = text;
+  const replacements = terms
+    .filter(
+      (term) =>
+        term.term.trim().length >= 2 &&
+        term.replacement?.trim() &&
+        term.replacement.trim() !== term.term.trim(),
+    )
+    .sort((a, b) => b.term.length - a.term.length);
+
+  for (const term of replacements) {
+    const source = escapeRegExp(term.term.trim());
+    const replacement = term.replacement?.trim() ?? "";
+    const pattern = new RegExp(source, "giu");
+    next = next.replace(pattern, (match, offset: number) => {
+      const start = offset;
+      const end = offset + match.length;
+      if (
+        !isSafeVoiceReplacementBoundary(next, start - 1, "before") ||
+        !isSafeVoiceReplacementBoundary(next, end, "after")
+      ) {
+        return match;
+      }
+      return replacement;
+    });
+  }
+
+  return next;
+}
+
+export function applyVoiceContextReplacements(
+  text: string,
+  contextPack: VoiceContextPack | undefined,
+): string {
+  const pack = sanitizeVoiceContextPack(contextPack);
+  return applyVoiceTermReplacements(text, pack?.terms);
+}

package/corpus/templates/analytics/AGENTS.md

@@ -113,9 +113,10 @@ details live in `.agents/skills/`.
 
 ## Session Replay
 
-- `/sessions` lists scoped first-party session recordings. Filters live in the
-  URL (`range`, `app`, `q`) and are mirrored through application state so
-  the agent sees the same list the user sees.
+- `/sessions` lists scoped, signed-in first-party session recordings with an
+  email-backed visitor identity and playable replay events. Filters live in the
+  URL (`range`, `app`, `q`) and are mirrored through application state so the
+  agent sees the same list the user sees.
 - `/sessions/:recordingId` loads summary/timeline data through
   `get-session-replay-summary` and fetches rrweb payloads only through scoped
   chunk routes. Do not expose storage/provider URLs, raw chunk table access, or

package/corpus/templates/analytics/actions/list-session-recordings.ts

@@ -27,7 +27,12 @@ export default defineAction({
     template: z.string().optional().describe("Optional template filter"),
     sessionId: z.string().optional().describe("Optional analytics session id"),
     userId: z.string().optional().describe("Optional signed-in user email"),
-    anonymousId: z.string().optional().describe("Optional anonymous user id"),
+    anonymousId: z
+      .string()
+      .optional()
+      .describe(
+        "Optional secondary anonymous id filter for otherwise email-backed recordings",
+      ),
     path: z.string().optional().describe("Optional exact path filter"),
     from: z
       .string()