dsh-her 0.0.1 → 0.1.0

package/lib/types/memory/digest.d.ts

@@ -0,0 +1,33 @@
+/**
+ * The daily digest: once a day, the day's conversation is read back from her
+ * DSH session and a model writes her diary page and picks out what is worth
+ * keeping. This file is the pure part: the transcript, the prompt, and a
+ * forgiving parser for the model's JSON.
+ */
+import type { DigestResult } from './store.ts';
+export interface TranscriptLine {
+    who: 'you' | 'her';
+    text: string;
+    time: number;
+}
+/** Events of her session within one local day, as "you" / "her" lines. Context notes and tags are left out. */
+export declare function transcriptOf(events: Array<{
+    type: string;
+    time?: number;
+    data: unknown;
+}>, dayStart: number, dayEnd: number): TranscriptLine[];
+export declare function digestPrompt(name: string, day: string, transcript: TranscriptLine[], knownFacts: string[], knownEvents?: string[], pronoun?: string): {
+    system: string;
+    user: string;
+};
+/**
+ * Right after a talk: what in these few turns is worth keeping, so it shows
+ * on the "她记得的事" page at once (the diary still waits for the day's digest).
+ * Same answer shape as the digest, without the diary; parseDigest reads it.
+ */
+export declare function notePrompt(name: string, day: string, transcript: TranscriptLine[], knownFacts: string[], knownEvents?: string[], pronoun?: string): {
+    system: string;
+    user: string;
+};
+/** The first JSON object in the model's answer, checked and cleaned; undefined if there is none. */
+export declare function parseDigest(answer: string): DigestResult | undefined;

package/lib/types/memory/search.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Recall without embeddings (docs/内置记忆.md §3): Chinese is cut into
+ * character bigrams (single characters for one-character runs), Latin text
+ * into lowercase words, scored with BM25, and newer memories weigh a little
+ * more. DeepSeek offers no embedding API and a local embedding model would
+ * add hundreds of MB, so this is enough for one person's memories.
+ */
+export interface SearchDoc {
+    id: string;
+    text: string;
+    /** When it was written or happened (ms). */
+    at: number;
+}
+export declare function tokens(text: string): string[];
+export declare function search(docs: SearchDoc[], query: string, now: number, limit?: number): Array<SearchDoc & {
+    score: number;
+}>;

package/lib/types/memory/store.d.ts

@@ -0,0 +1,132 @@
+/**
+ * Her built-in memory (docs/内置记忆.md): plain files under
+ * `<home>/characters/<cid>/memory/`, readable and editable by hand.
+ *
+ * - facts.json     about you: name, likes, dislikes, people, dates. Facts she
+ *                  was told to remember are confirmed at once; facts found by
+ *                  the daily digest are candidates, confirmed by you or after
+ *                  7 days, except sensitive ones (health, money, others'
+ *                  privacy), which wait for you.
+ * - episodes.jsonl things worth bringing up later, some with a date: on that
+ *                  day and the day before, they are put in front of her.
+ * - diary/<day>.md her diary, one page a day, written by the digest.
+ *
+ * `MemoryProvider` is the seam where the soul engine (dsh-vael) can replace
+ * all of this later. Writes go to a temporary file first and are renamed.
+ */
+export interface Fact {
+    id: string;
+    text: string;
+    status: 'confirmed' | 'candidate';
+    sensitive: boolean;
+    source: 'remember' | 'digest';
+    createdAt: string;
+}
+export interface Episode {
+    id: string;
+    text: string;
+    /** When it was said or happened. */
+    at: string;
+    /** The day it is about (YYYY-MM-DD), if any. */
+    due?: string;
+}
+export interface MemoryHit {
+    id: string;
+    kind: 'fact' | 'episode' | 'diary';
+    text: string;
+    at: string;
+}
+export interface MemoryProvider {
+    /** "About you" for her prompt (at most ~300 characters). */
+    profile(): string;
+    /** The last two days of her diary, short. */
+    recent(now: Date): string;
+    /** Things with a date of today or tomorrow. */
+    due(now: Date): Episode[];
+    remember(item: {
+        kind: 'fact' | 'episode';
+        text: string;
+        due?: string;
+        sensitive?: boolean;
+        source?: Fact['source'];
+    }, now: Date): MemoryHit;
+    recall(query: string, now: Date, limit?: number): MemoryHit[];
+    list(): {
+        facts: Fact[];
+        episodes: Episode[];
+        diary: string[];
+    };
+    remove(ids: string[]): number;
+    confirm(ids: string[]): number;
+    /** Candidates older than 7 days and not sensitive become facts; returns how many. */
+    settle(now: Date): number;
+    /** Today's diary and what the digest found. */
+    applyDigest(day: string, digest: DigestResult, now: Date): {
+        facts: number;
+        episodes: number;
+    };
+    keep(found: Pick<DigestResult, 'facts' | 'episodes'>, now: Date): {
+        facts: number;
+        episodes: number;
+    };
+    lastDigestDay(): string | undefined;
+    exportMarkdown(): string;
+    clear(): void;
+}
+export interface DigestResult {
+    diary: string;
+    facts: Array<{
+        text: string;
+        sensitive?: boolean;
+    }>;
+    episodes: Array<{
+        text: string;
+        due?: string;
+    }>;
+}
+export declare function dayOf(date: Date): string;
+export declare class FileMemory implements MemoryProvider {
+    private readonly dir;
+    constructor(dir: string);
+    private read;
+    private write;
+    private facts;
+    private saveFacts;
+    private episodes;
+    private saveEpisodes;
+    private diaryDays;
+    private nextId;
+    profile(): string;
+    recent(now: Date): string;
+    due(now: Date): Episode[];
+    remember(item: {
+        kind: 'fact' | 'episode';
+        text: string;
+        due?: string;
+        sensitive?: boolean;
+        source?: Fact['source'];
+    }, now: Date): MemoryHit;
+    recall(query: string, now: Date, limit?: number): MemoryHit[];
+    list(): {
+        facts: Fact[];
+        episodes: Episode[];
+        diary: string[];
+    };
+    remove(ids: string[]): number;
+    /** Drop diary sentences that mostly repeat any of these texts. */
+    private redactDiary;
+    confirm(ids: string[]): number;
+    settle(now: Date): number;
+    applyDigest(day: string, digest: DigestResult, now: Date): {
+        facts: number;
+        episodes: number;
+    };
+    /** Keep what a digest or a note found: new facts as candidates, events not already written down. */
+    keep(found: Pick<DigestResult, 'facts' | 'episodes'>, now: Date): {
+        facts: number;
+        episodes: number;
+    };
+    lastDigestDay(): string | undefined;
+    exportMarkdown(): string;
+    clear(): void;
+}

package/lib/types/protocol/index.d.ts

@@ -0,0 +1,434 @@
+/**
+ * Body protocol v1: the messages between the brain (this plugin) and a body
+ * (the Electron desktop app, later Unity or a phone). Design: docs/身体协议.md.
+ *
+ * Text frames carry one JSON message each. Binary frames carry audio: a 4-byte
+ * big-endian header holding the stream id, then the payload. The audio frame
+ * is sent first, then the `speak.seg` or `user.utterance` naming its stream.
+ *
+ * This file has no runtime dependencies so the body can import the same source.
+ */
+export declare const PROTOCOL_VERSION = 1;
+export type AvatarState = 'idle' | 'listen' | 'think' | 'talk' | 'sleep';
+export type WindowMode = 'overlay' | 'wallpaper' | 'window';
+export type AudioFormat = 'wav' | 'mp3' | 'opus';
+export interface CharMark {
+    /** Character offset in the segment text. */
+    i: number;
+    /** Milliseconds from the start of the segment audio. */
+    ms: number;
+}
+/** A performance cue riding on a sentence: applied when that sentence starts playing. */
+export interface Cue {
+    kind: 'emotion' | 'act';
+    name: string;
+}
+export interface PackAction {
+    id: string;
+    desc: string;
+    durationMs?: number;
+    hasAudio?: boolean;
+    transcript?: string;
+}
+export interface PackSummary {
+    packId: string;
+    kind: 'video';
+    name: string;
+    layer: 'overlay' | 'scene';
+    states: AvatarState[];
+    emotions: string[];
+    actions: PackAction[];
+    hitParts: string[];
+}
+/**
+ * A look on the body: one character in several scenes and how they switch
+ * (docs/形象工坊与多场景.md §5). Its pack id in `pack.list` is `look:<id>`.
+ */
+export interface LookView {
+    id: string;
+    name: string;
+    scenes: Array<{
+        id: string;
+        name: string;
+        when: string;
+        on: boolean;
+    }>;
+    current: string;
+    /** Switching by time of day and by whether you are around. */
+    auto: boolean;
+    /** Held in the current scene until asked to move. */
+    locked: boolean;
+    /** It has a desk-pet version ("浮在最上层" shows that instead of the scene). */
+    desk: boolean;
+}
+export interface BodyCaps {
+    audioOut: AudioFormat[];
+    micIn?: {
+        formats: string[];
+        vad?: string;
+    };
+    asrLocal: boolean;
+    ttsLocal: string[];
+    modes: WindowMode[];
+    screens: Array<{
+        id: number;
+        w: number;
+        h: number;
+        scale: number;
+    }>;
+}
+export interface CharacterView {
+    cid: string;
+    name: string;
+    packId?: string;
+    mode: WindowMode;
+    subtitle?: {
+        color?: string;
+    };
+}
+export interface Hello {
+    t: 'hello';
+    v: number;
+    token: string;
+    body: {
+        kind: 'video' | 'vrm' | 'unity' | 'web';
+        app: string;
+        version: string;
+        os: string;
+    };
+    caps: BodyCaps;
+    packs: PackSummary[];
+}
+export type BodyMessage = Hello | {
+    t: 'ping';
+    at: number;
+} | {
+    t: 'user.text';
+    cid?: string;
+    text: string;
+} | {
+    t: 'user.utterance';
+    cid?: string;
+    uttId: string;
+    format: string;
+    durationMs: number;
+    stream?: number;
+    text?: string;
+} | {
+    t: 'user.speechStart';
+    cid?: string;
+} | {
+    t: 'interrupt';
+    cid: string;
+    uid: string;
+    seq: number;
+    playedMs: number;
+} | {
+    t: 'speak.progress';
+    cid: string;
+    uid: string;
+    seq: number;
+    phase: 'start' | 'end';
+} | {
+    t: 'act.done';
+    cid: string;
+    id: string;
+    ok: boolean;
+    reason?: string;
+} | {
+    t: 'answer';
+    cid: string;
+    id: string;
+    option?: string;
+    text?: string;
+} | {
+    t: 'touch';
+    cid: string;
+    part: string;
+    kind: 'tap' | 'double' | 'long';
+} | {
+    t: 'drag';
+    cid: string;
+    phase: 'start' | 'end';
+    pos: {
+        x: number;
+        y: number;
+    };
+} | {
+    t: 'presence';
+    state: 'active' | 'idle' | 'away';
+    fullscreen: boolean;
+    locked: boolean;
+} | {
+    t: 'posture';
+    cid: string;
+    anchor: string;
+} | {
+    t: 'mode.changed';
+    cid: string;
+    mode: WindowMode;
+    pos?: {
+        x: number;
+        y: number;
+        screen?: number;
+    };
+} | {
+    t: 'pack.progress';
+    id: string;
+    done: number;
+    total: number;
+    step: string;
+} | {
+    t: 'pack.report';
+    id: string;
+    ok: boolean;
+    pack?: PackSummary;
+    lines: Array<{
+        level: 'ok' | 'warn' | 'error';
+        text: string;
+    }>;
+}
+/** The body's settings window wants who she is and what she remembers (answered with `panel`). */
+ | {
+    t: 'panel.get';
+    cid: string;
+}
+/** Changes made in the body's settings window; answered with a fresh `panel`. */
+ | {
+    t: 'panel.set';
+    cid: string;
+    character?: Partial<PanelCharacter>;
+    forget?: string[];
+    confirm?: string[];
+}
+/** The packs this body has imported, and the one on screen. */
+ | {
+    t: 'pack.list';
+    packs: PackSummary[];
+    active?: string;
+}
+/** The look on screen and its scenes (null: a single pack, no look). */
+ | {
+    t: 'look.state';
+    look: LookView | null;
+}
+/** Answer to `folder.pick`: what the user chose (none: cancelled). */
+ | {
+    t: 'folder.picked';
+    id: string;
+    path?: string;
+} | {
+    t: 'error';
+    code: string;
+    message: string;
+};
+export interface Welcome {
+    t: 'welcome';
+    v: number;
+    sessionId: string;
+    characters: CharacterView[];
+    config: {
+        mic: 'push-to-talk' | 'always';
+        bargeIn: boolean;
+    };
+}
+/** What the body's settings window can show and change about her. */
+export interface PanelCharacter {
+    name: string;
+    persona: string;
+    callYou: string;
+    pronoun: '他' | '她' | 'TA';
+}
+export interface PanelMemory {
+    facts: Array<{
+        id: string;
+        text: string;
+        status: 'confirmed' | 'candidate';
+        sensitive: boolean;
+        createdAt: string;
+    }>;
+    episodes: Array<{
+        id: string;
+        text: string;
+        at: string;
+        due?: string;
+    }>;
+    diary: string[];
+}
+export type BrainMessage = Welcome | {
+    t: 'reject';
+    reason: 'version' | 'token';
+    message: string;
+} | {
+    t: 'pong';
+    at: number;
+} | {
+    t: 'character.load';
+    cid: string;
+    name: string;
+    packId?: string;
+    mode: WindowMode;
+    subtitle?: {
+        color?: string;
+    };
+} | {
+    t: 'character.unload';
+    cid: string;
+} | {
+    t: 'state';
+    cid: string;
+    state: AvatarState;
+} | {
+    t: 'emotion';
+    cid: string;
+    emotion: string;
+    intensity?: number;
+} | {
+    t: 'speak.begin';
+    cid: string;
+    uid: string;
+    emotion?: string;
+} | {
+    t: 'speak.seg';
+    cid: string;
+    uid: string;
+    seq: number;
+    text: string;
+    durationMs: number;
+    format?: AudioFormat;
+    stream?: number;
+    marks?: CharMark[];
+    cues?: Cue[];
+} | {
+    t: 'speak.end';
+    cid: string;
+    uid: string;
+} | {
+    t: 'speak.cancel';
+    cid: string;
+    uid: string;
+} | {
+    t: 'act';
+    cid: string;
+    id: string;
+    action: string;
+    args?: unknown;
+} | {
+    t: 'bubble';
+    cid: string;
+    text: string;
+    ttlMs?: number;
+}
+/**
+ * What the brain recognized in a `user.utterance` (empty: nothing understood).
+ * `ignored`: right after a barge-in it was her own voice (echo) or only a
+ * filler sound, so it was dropped and does not count as your turn.
+ */
+ | {
+    t: 'heard';
+    cid: string;
+    uttId: string;
+    text: string;
+    ignored?: 'echo' | 'filler';
+} | {
+    t: 'ask';
+    cid: string;
+    id: string;
+    question: string;
+    options: string[];
+} | {
+    t: 'mode';
+    cid: string;
+    mode: WindowMode;
+    screen?: number;
+    pos?: {
+        x: number;
+        y: number;
+    };
+    scale?: number;
+} | {
+    t: 'config';
+    mic?: 'push-to-talk' | 'always';
+    bargeIn?: boolean;
+}
+/** Let the user choose a folder on the body's computer and import it (progress and report follow). */
+ | {
+    t: 'pack.pick';
+    id: string;
+} | {
+    t: 'pack.import';
+    id: string;
+    dir: string;
+}
+/** Show this imported pack (or `look:<id>`) from now on. */
+ | {
+    t: 'pack.use';
+    packId: string;
+}
+/** Change how the look on screen switches scenes, or go to one now. */
+ | {
+    t: 'look.set';
+    auto?: boolean;
+    locked?: boolean;
+    go?: string;
+    on?: Record<string, boolean>;
+}
+/**
+ * Import a 形象工坊 project (folders on the body's computer): each part with
+ * clips becomes a pack, tied together as look `id`, and shown. Progress and
+ * the report come back as `pack.progress` / `pack.report`.
+ */
+ | {
+    t: 'look.import';
+    id: string;
+    name: string;
+    dir: string;
+    desk?: string;
+    default?: string;
+    scenes: Array<{
+        id: string;
+        name: string;
+        folder: string;
+        when?: {
+            hours?: string;
+            user?: 'active' | 'idle';
+        };
+    }>;
+}
+/** Open 做场景 on a project's scene folder: its plate.png is the photo, scene.json and crop.png are written there. */
+ | {
+    t: 'scene.frame';
+    dir: string;
+    name: string;
+}
+/**
+ * Let the user choose a folder (or, with `file`, also a file of that
+ * extension) on the body's computer; answered with `folder.picked`.
+ */
+ | {
+    t: 'folder.pick';
+    id: string;
+    title: string;
+    button: string;
+    file?: string;
+}
+/** Who she is and what she remembers, for the body's settings window; `error` when a change was refused. */
+ | {
+    t: 'panel';
+    cid: string;
+    character: PanelCharacter;
+    memory: PanelMemory;
+    error?: string;
+};
+export type AnyMessage = BodyMessage | BrainMessage;
+/** Parse one text frame. Unknown `t` values pass through so newer peers degrade instead of failing. */
+export declare function parseMessage<T extends {
+    t: string;
+} = AnyMessage>(raw: string): T | undefined;
+/** Prefix an audio payload with its stream id. */
+export declare function encodeAudioFrame(stream: number, payload: Uint8Array): Uint8Array;
+/** Split a binary frame into its stream id and payload; undefined when it is too short. */
+export declare function decodeAudioFrame(frame: Uint8Array): {
+    stream: number;
+    payload: Uint8Array;
+} | undefined;

package/lib/types/server/brain-server.d.ts

@@ -0,0 +1,71 @@
+/**
+ * The brain's side of the body protocol: a WebSocket server on loopback.
+ * Design: docs/身体协议.md §2–3.
+ *
+ * - Port: the preferred one, else the next free ones, else any free port.
+ * - Token: created once and kept in `<home>/token` (mode 600); a body proves
+ *   itself with it in `hello`, never in the URL.
+ * - Discovery: `<home>/brain.json` (mode 600) says where the brain listens and
+ *   carries the token, so a body started by hand can find it; it is rewritten
+ *   on every start because the port may change, and removed on stop.
+ * - Heartbeat: a body pings every 15 s; one silent for 45 s is dropped.
+ * - Audio: a binary frame (4-byte stream id + payload) comes first, then the
+ *   JSON message naming its stream; the server pairs them, both directions.
+ *
+ * The server knows nothing about DSH or sessions; the plugin wires it.
+ */
+import { type BodyMessage, type BrainMessage, type Hello } from '../protocol/index.ts';
+export declare const DEFAULT_PORT = 38100;
+export interface BodyClient {
+    readonly id: number;
+    readonly hello: Hello;
+    send(message: BrainMessage): void;
+    /** Send audio for a stream; the message naming the stream must follow. */
+    sendAudio(stream: number, payload: Uint8Array): void;
+}
+export interface BrainServerOptions {
+    /** `<home>/token` and `<home>/brain.json`. */
+    tokenFile: string;
+    discoveryFile: string;
+    preferredPort?: number;
+    /** Called once a body has said a valid hello; return its welcome. */
+    welcome(client: BodyClient): BrainMessage;
+    /** Every later message from an admitted body, with the audio of its `stream` if it named one. */
+    message(client: BodyClient, message: BodyMessage, audio?: Uint8Array): void;
+    /** An admitted body went away. */
+    closed?(client: BodyClient): void;
+    logger?: {
+        info(message: string): void;
+        warn(message: string): void;
+    };
+}
+export interface Discovery {
+    url: string;
+    token: string;
+    pid: number;
+    protocol: number;
+    startedAt: string;
+}
+/** The brain's token: read, or created on first start. */
+export declare function loadToken(file: string): string;
+export declare class BrainServer {
+    private server;
+    private token;
+    private nextId;
+    private readonly clients;
+    private reaper;
+    private readonly lastSeen;
+    private readonly options;
+    constructor(options: BrainServerOptions);
+    get port(): number | undefined;
+    get url(): string | undefined;
+    get bodies(): BodyClient[];
+    start(): Promise<Discovery>;
+    broadcast(message: BrainMessage): void;
+    /** Send audio and then the message naming its stream to every body. */
+    broadcastWithAudio(message: BrainMessage, stream: number, payload: Uint8Array): void;
+    stop(): Promise<void>;
+    private accept;
+    private admit;
+    private reap;
+}

package/lib/types/voice/cloud-key.d.ts

@@ -0,0 +1,10 @@
+/**
+ * The Model Studio (百炼) API key, never kept in a file: the environment
+ * variable `DASHSCOPE_API_KEY`, or on macOS the keychain item the user adds with
+ *
+ *   security add-generic-password -a dsh-her -s dsh-her-dashscope -w
+ *
+ * Read on every connection, so a key added later works without restarting DSH.
+ */
+export declare const KEYCHAIN_SERVICE = "dsh-her-dashscope";
+export declare function readCloudKey(): Promise<string | undefined>;