@phuetz/code-buddy 1.2.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +56 -3
- package/dist/agent/autonomous/agentic-coding-contract.d.ts +6 -6
- package/dist/agent/base-agent.d.ts +4 -0
- package/dist/agent/base-agent.js +6 -0
- package/dist/agent/facades/infrastructure-facade.d.ts +9 -2
- package/dist/agent/facades/infrastructure-facade.js +15 -6
- package/dist/agent/self-improvement/authored-artifact-gate.d.ts +18 -0
- package/dist/agent/self-improvement/authored-artifact-gate.js +42 -0
- package/dist/agent/self-improvement/authored-tool-runtime.d.ts +27 -0
- package/dist/agent/self-improvement/authored-tool-runtime.js +57 -0
- package/dist/agent/self-improvement/authored-tool-store.d.ts +24 -0
- package/dist/agent/self-improvement/authored-tool-store.js +57 -0
- package/dist/agent/self-improvement/llm-tool-proposer.d.ts +41 -0
- package/dist/agent/self-improvement/llm-tool-proposer.js +136 -0
- package/dist/agent/self-improvement/sandbox-scorer.d.ts +17 -0
- package/dist/agent/self-improvement/sandbox-scorer.js +43 -0
- package/dist/agent/self-improvement/self-knowledge.d.ts +8 -0
- package/dist/agent/self-improvement/self-knowledge.js +24 -0
- package/dist/agent/self-improvement/skill-benchmark.d.ts +9 -0
- package/dist/agent/self-improvement/skill-benchmark.js +22 -0
- package/dist/agent/self-improvement/skill-consolidator.d.ts +71 -0
- package/dist/agent/self-improvement/skill-consolidator.js +137 -0
- package/dist/agent/self-improvement/skill-engine.d.ts +42 -0
- package/dist/agent/self-improvement/skill-engine.js +87 -0
- package/dist/agent/self-improvement/skill-gate.d.ts +19 -0
- package/dist/agent/self-improvement/skill-gate.js +62 -0
- package/dist/agent/self-improvement/skill-mutator.d.ts +74 -0
- package/dist/agent/self-improvement/skill-mutator.js +223 -0
- package/dist/agent/self-improvement/skill-proposer.d.ts +40 -0
- package/dist/agent/self-improvement/skill-proposer.js +82 -0
- package/dist/agent/self-improvement/skill-types.d.ts +41 -0
- package/dist/agent/self-improvement/skill-types.js +13 -0
- package/dist/agent/self-improvement/tool-benchmark.d.ts +10 -0
- package/dist/agent/self-improvement/tool-benchmark.js +37 -0
- package/dist/agent/self-improvement/tool-engine.d.ts +54 -0
- package/dist/agent/self-improvement/tool-engine.js +101 -0
- package/dist/agent/self-improvement/tool-gate.d.ts +20 -0
- package/dist/agent/self-improvement/tool-gate.js +78 -0
- package/dist/agent/self-improvement/tool-proposer.d.ts +31 -0
- package/dist/agent/self-improvement/tool-proposer.js +34 -0
- package/dist/agent/self-improvement/tool-skill-mutator.d.ts +40 -0
- package/dist/agent/self-improvement/tool-skill-mutator.js +79 -0
- package/dist/agent/self-improvement/tool-types.d.ts +48 -0
- package/dist/agent/self-improvement/tool-types.js +9 -0
- package/dist/agent/self-improvement/types.d.ts +3 -1
- package/dist/agent/tool-handler.js +3 -0
- package/dist/codebuddy/providers/provider-chatgpt-responses.js +6 -1
- package/dist/codebuddy/tools.d.ts +7 -0
- package/dist/codebuddy/tools.js +40 -0
- package/dist/commands/cli/improve-command.js +123 -0
- package/dist/commands/enhanced-command-handler.js +1 -1
- package/dist/commands/handlers/missing-handlers.d.ts +1 -1
- package/dist/commands/handlers/missing-handlers.js +26 -3
- package/dist/commands/skills-cli/index.js +123 -0
- package/dist/commands/slash/builtin-commands.js +1 -1
- package/dist/companion/percepts.js +11 -1
- package/dist/context/bootstrap-loader.js +6 -23
- package/dist/context/import-directive-parser.d.ts +4 -0
- package/dist/context/import-directive-parser.js +51 -6
- package/dist/context/instruction-excludes.d.ts +30 -1
- package/dist/context/instruction-excludes.js +71 -1
- package/dist/context/jit-context.d.ts +8 -10
- package/dist/context/jit-context.js +28 -106
- package/dist/context/project-context.d.ts +90 -0
- package/dist/context/project-context.js +295 -0
- package/dist/daemon/autonomous-loop.d.ts +31 -1
- package/dist/daemon/autonomous-loop.js +80 -2
- package/dist/harness/contract.d.ts +28 -28
- package/dist/identity/identity-manager.js +3 -2
- package/dist/index.js +17 -1
- package/dist/mcp/mcp-resources.js +2 -3
- package/dist/sensory/dreaming.d.ts +45 -0
- package/dist/sensory/dreaming.js +114 -0
- package/dist/sensory/heartbeat-scheduler.d.ts +38 -0
- package/dist/sensory/heartbeat-scheduler.js +72 -0
- package/dist/sensory/reactions.d.ts +24 -0
- package/dist/sensory/reactions.js +31 -0
- package/dist/sensory/screen-reaction.d.ts +23 -0
- package/dist/sensory/screen-reaction.js +59 -0
- package/dist/sensory/sensory-bridge.d.ts +23 -0
- package/dist/sensory/sensory-bridge.js +85 -0
- package/dist/sensory/sensory-memory.d.ts +20 -0
- package/dist/sensory/sensory-memory.js +39 -0
- package/dist/sensory/speech-reaction.d.ts +21 -0
- package/dist/sensory/speech-reaction.js +83 -0
- package/dist/sensory/vision-reaction.d.ts +31 -0
- package/dist/sensory/vision-reaction.js +74 -0
- package/dist/server/index.js +89 -0
- package/dist/services/prompt-builder.d.ts +10 -0
- package/dist/services/prompt-builder.js +75 -9
- package/dist/skills/parser.js +3 -0
- package/dist/skills/skill-importer.d.ts +58 -0
- package/dist/skills/skill-importer.js +261 -0
- package/dist/skills/skill-sources.d.ts +20 -0
- package/dist/skills/skill-sources.js +102 -0
- package/dist/skills/types.d.ts +6 -0
- package/dist/tools/register-tool-handler.d.ts +25 -0
- package/dist/tools/register-tool-handler.js +100 -0
- package/dist/tools/registry.d.ts +6 -0
- package/dist/tools/registry.js +8 -0
- package/dist/utils/init-project.d.ts +7 -0
- package/dist/utils/init-project.js +37 -0
- package/dist/utils/settings-manager.d.ts +12 -0
- package/package.json +1 -1
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Heartbeat scheduler — turns the nervous-system heartbeat into a PACEMAKER for
|
|
3
|
+
* periodic processing in Code Buddy. Register "treatments" that fire every N
|
|
4
|
+
* beats; the daemon's heart rate (BUDDY_SENSE_HEARTBEAT_MS) dials how often they
|
|
5
|
+
* run — a faster heart triggers more processing, a slower one less.
|
|
6
|
+
*
|
|
7
|
+
* This is the heartbeat-paced analogue of Hermes's curator/cron and OpenClaw's
|
|
8
|
+
* cron-jobs + dreaming: scheduled background work, but driven by the heart rather
|
|
9
|
+
* than a wall clock. Each beat carries vital signs (uptime, load) the treatments
|
|
10
|
+
* can read — so a treatment can also adapt to the body's state.
|
|
11
|
+
*
|
|
12
|
+
* @module sensory/heartbeat-scheduler
|
|
13
|
+
*/
|
|
14
|
+
export interface HeartbeatContext {
|
|
15
|
+
beat: number;
|
|
16
|
+
uptimeMs?: number;
|
|
17
|
+
load1?: number | null;
|
|
18
|
+
}
|
|
19
|
+
export type HeartbeatHandler = (ctx: HeartbeatContext) => void | Promise<void>;
|
|
20
|
+
export interface HeartbeatTask {
|
|
21
|
+
name: string;
|
|
22
|
+
/** Fire when `beat % everyBeats === 0` (1 = every beat). */
|
|
23
|
+
everyBeats: number;
|
|
24
|
+
handler: HeartbeatHandler;
|
|
25
|
+
}
|
|
26
|
+
export declare class HeartbeatScheduler {
|
|
27
|
+
private readonly tasks;
|
|
28
|
+
private listenerId?;
|
|
29
|
+
/** Register a treatment to run every N beats. Re-registering replaces it. */
|
|
30
|
+
register(task: HeartbeatTask): void;
|
|
31
|
+
unregister(name: string): void;
|
|
32
|
+
list(): HeartbeatTask[];
|
|
33
|
+
/** Subscribe to the sensory bus and start pacing. Idempotent. */
|
|
34
|
+
start(): void;
|
|
35
|
+
stop(): void;
|
|
36
|
+
private onBeat;
|
|
37
|
+
}
|
|
38
|
+
export declare function getHeartbeatScheduler(): HeartbeatScheduler;
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Heartbeat scheduler — turns the nervous-system heartbeat into a PACEMAKER for
|
|
3
|
+
* periodic processing in Code Buddy. Register "treatments" that fire every N
|
|
4
|
+
* beats; the daemon's heart rate (BUDDY_SENSE_HEARTBEAT_MS) dials how often they
|
|
5
|
+
* run — a faster heart triggers more processing, a slower one less.
|
|
6
|
+
*
|
|
7
|
+
* This is the heartbeat-paced analogue of Hermes's curator/cron and OpenClaw's
|
|
8
|
+
* cron-jobs + dreaming: scheduled background work, but driven by the heart rather
|
|
9
|
+
* than a wall clock. Each beat carries vital signs (uptime, load) the treatments
|
|
10
|
+
* can read — so a treatment can also adapt to the body's state.
|
|
11
|
+
*
|
|
12
|
+
* @module sensory/heartbeat-scheduler
|
|
13
|
+
*/
|
|
14
|
+
import { getGlobalEventBus } from '../events/event-bus.js';
|
|
15
|
+
import { logger } from '../utils/logger.js';
|
|
16
|
+
export class HeartbeatScheduler {
|
|
17
|
+
tasks = new Map();
|
|
18
|
+
listenerId;
|
|
19
|
+
/** Register a treatment to run every N beats. Re-registering replaces it. */
|
|
20
|
+
register(task) {
|
|
21
|
+
if (!Number.isInteger(task.everyBeats) || task.everyBeats < 1) {
|
|
22
|
+
throw new Error(`heartbeat task "${task.name}": everyBeats must be an integer >= 1`);
|
|
23
|
+
}
|
|
24
|
+
this.tasks.set(task.name, task);
|
|
25
|
+
}
|
|
26
|
+
unregister(name) {
|
|
27
|
+
this.tasks.delete(name);
|
|
28
|
+
}
|
|
29
|
+
list() {
|
|
30
|
+
return [...this.tasks.values()];
|
|
31
|
+
}
|
|
32
|
+
/** Subscribe to the sensory bus and start pacing. Idempotent. */
|
|
33
|
+
start() {
|
|
34
|
+
if (this.listenerId)
|
|
35
|
+
return;
|
|
36
|
+
const bus = getGlobalEventBus();
|
|
37
|
+
this.listenerId = bus.on('sensory:perception', (evt) => {
|
|
38
|
+
const m = evt.metadata;
|
|
39
|
+
if (m?.modality !== 'vital' || m?.kind !== 'heartbeat')
|
|
40
|
+
return;
|
|
41
|
+
const beat = Number(m.payload?.beat ?? 0);
|
|
42
|
+
if (!Number.isFinite(beat) || beat < 1)
|
|
43
|
+
return;
|
|
44
|
+
void this.onBeat({ beat, uptimeMs: m.payload?.uptime_ms, load1: m.payload?.load1 });
|
|
45
|
+
});
|
|
46
|
+
}
|
|
47
|
+
stop() {
|
|
48
|
+
if (this.listenerId) {
|
|
49
|
+
getGlobalEventBus().off(this.listenerId);
|
|
50
|
+
this.listenerId = undefined;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
async onBeat(ctx) {
|
|
54
|
+
for (const task of this.tasks.values()) {
|
|
55
|
+
if (ctx.beat % task.everyBeats !== 0)
|
|
56
|
+
continue;
|
|
57
|
+
try {
|
|
58
|
+
await task.handler(ctx);
|
|
59
|
+
}
|
|
60
|
+
catch (err) {
|
|
61
|
+
logger.warn(`[heartbeat] treatment "${task.name}" failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
let singleton;
|
|
67
|
+
export function getHeartbeatScheduler() {
|
|
68
|
+
if (!singleton)
|
|
69
|
+
singleton = new HeartbeatScheduler();
|
|
70
|
+
return singleton;
|
|
71
|
+
}
|
|
72
|
+
//# sourceMappingURL=heartbeat-scheduler.js.map
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Sensory reactions — subscribers that turn `sensory:perception` events from the
|
|
3
|
+
* nervous-system daemon into Code Buddy behaviour. Phase 1 logs + exposes a hook;
|
|
4
|
+
* Phase 2 will route (speech_end → STT → turn; motion → camera_analyze).
|
|
5
|
+
*
|
|
6
|
+
* @module sensory/reactions
|
|
7
|
+
*/
|
|
8
|
+
import type { BaseEvent } from '../events/types.js';
|
|
9
|
+
export interface Perception {
|
|
10
|
+
modality?: string;
|
|
11
|
+
kind?: string;
|
|
12
|
+
salience?: number;
|
|
13
|
+
/** Sense-relative timestamp (frame-relative for audio, unix-millis for vital). */
|
|
14
|
+
tsMs?: number;
|
|
15
|
+
/** Ingest wall-clock (one consistent clock across senses — used for dream windows). */
|
|
16
|
+
receivedAt?: number;
|
|
17
|
+
payload?: unknown;
|
|
18
|
+
}
|
|
19
|
+
export declare function perceptionOf(evt: BaseEvent): Perception;
|
|
20
|
+
/**
|
|
21
|
+
* Wire reactions to the sensory bus. Returns an unsubscribe fn. `onPerception`
|
|
22
|
+
* is an injectable hook (tests + future routing).
|
|
23
|
+
*/
|
|
24
|
+
export declare function wireSensoryReactions(onPerception?: (p: Perception, evt: BaseEvent) => void): () => void;
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Sensory reactions — subscribers that turn `sensory:perception` events from the
|
|
3
|
+
* nervous-system daemon into Code Buddy behaviour. Phase 1 logs + exposes a hook;
|
|
4
|
+
* Phase 2 will route (speech_end → STT → turn; motion → camera_analyze).
|
|
5
|
+
*
|
|
6
|
+
* @module sensory/reactions
|
|
7
|
+
*/
|
|
8
|
+
import { getGlobalEventBus } from '../events/event-bus.js';
|
|
9
|
+
import { logger } from '../utils/logger.js';
|
|
10
|
+
import { getSensoryMemory } from './sensory-memory.js';
|
|
11
|
+
export function perceptionOf(evt) {
|
|
12
|
+
const meta = evt.metadata ?? {};
|
|
13
|
+
return { ...meta, receivedAt: evt.timestamp };
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Wire reactions to the sensory bus. Returns an unsubscribe fn. `onPerception`
|
|
17
|
+
* is an injectable hook (tests + future routing).
|
|
18
|
+
*/
|
|
19
|
+
export function wireSensoryReactions(onPerception) {
|
|
20
|
+
const bus = getGlobalEventBus();
|
|
21
|
+
const id = bus.on('sensory:perception', (evt) => {
|
|
22
|
+
const p = perceptionOf(evt);
|
|
23
|
+
logger.info(`[sensory] ${p.modality}/${p.kind} (salience ${p.salience ?? 0})`);
|
|
24
|
+
getSensoryMemory().push(p); // short-term memory → consolidated by dreaming
|
|
25
|
+
onPerception?.(p, evt);
|
|
26
|
+
});
|
|
27
|
+
return () => {
|
|
28
|
+
bus.off(id);
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
//# sourceMappingURL=reactions.js.map
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Screen reaction — on a `screen/change` event from the nervous-system daemon,
|
|
3
|
+
* record a companion percept and (optionally) run an injected analyzer (OCR /
|
|
4
|
+
* vision describe). DEBOUNCED, opt-in (`CODEBUDDY_SENSORY_SCREEN=true`),
|
|
5
|
+
* never-throws. The heavy OCR/describe is pluggable so we don't duplicate Code
|
|
6
|
+
* Buddy's existing OCR/vision; inject an analyzer to wire it.
|
|
7
|
+
*
|
|
8
|
+
* @module sensory/screen-reaction
|
|
9
|
+
*/
|
|
10
|
+
export interface ScreenAnalysis {
|
|
11
|
+
description?: string;
|
|
12
|
+
}
|
|
13
|
+
export interface ScreenAnalyzer {
|
|
14
|
+
analyze(): Promise<ScreenAnalysis>;
|
|
15
|
+
}
|
|
16
|
+
export interface ScreenReactionOptions {
|
|
17
|
+
/** Injectable OCR/vision describe (default: none — just record the change). */
|
|
18
|
+
analyzer?: ScreenAnalyzer;
|
|
19
|
+
debounceMs?: number;
|
|
20
|
+
cwd?: string;
|
|
21
|
+
now?: () => number;
|
|
22
|
+
}
|
|
23
|
+
export declare function wireScreenReaction(options?: ScreenReactionOptions): () => void;
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Screen reaction — on a `screen/change` event from the nervous-system daemon,
|
|
3
|
+
* record a companion percept and (optionally) run an injected analyzer (OCR /
|
|
4
|
+
* vision describe). DEBOUNCED, opt-in (`CODEBUDDY_SENSORY_SCREEN=true`),
|
|
5
|
+
* never-throws. The heavy OCR/describe is pluggable so we don't duplicate Code
|
|
6
|
+
* Buddy's existing OCR/vision; inject an analyzer to wire it.
|
|
7
|
+
*
|
|
8
|
+
* @module sensory/screen-reaction
|
|
9
|
+
*/
|
|
10
|
+
import { getGlobalEventBus } from '../events/event-bus.js';
|
|
11
|
+
import { logger } from '../utils/logger.js';
|
|
12
|
+
import { perceptionOf } from './reactions.js';
|
|
13
|
+
export function wireScreenReaction(options = {}) {
|
|
14
|
+
const bus = getGlobalEventBus();
|
|
15
|
+
const debounceMs = options.debounceMs ?? Number(process.env.CODEBUDDY_SCREEN_DEBOUNCE_MS ?? 5000);
|
|
16
|
+
const now = options.now ?? (() => Date.now());
|
|
17
|
+
let lastAt = Number.NEGATIVE_INFINITY;
|
|
18
|
+
let inFlight = false;
|
|
19
|
+
const id = bus.on('sensory:perception', (evt) => {
|
|
20
|
+
const p = perceptionOf(evt);
|
|
21
|
+
if (p.modality !== 'screen' || p.kind !== 'change')
|
|
22
|
+
return;
|
|
23
|
+
const t = now();
|
|
24
|
+
if (t - lastAt < debounceMs) {
|
|
25
|
+
logger.info('[screen] change (debounced)');
|
|
26
|
+
return;
|
|
27
|
+
}
|
|
28
|
+
if (inFlight)
|
|
29
|
+
return; // a prior analyze() is still running — don't stampede
|
|
30
|
+
lastAt = t;
|
|
31
|
+
inFlight = true;
|
|
32
|
+
void (async () => {
|
|
33
|
+
try {
|
|
34
|
+
const analysis = options.analyzer ? await options.analyzer.analyze() : {};
|
|
35
|
+
const { recordCompanionPercept } = await import('../companion/percepts.js');
|
|
36
|
+
const score = p.payload?.score;
|
|
37
|
+
await recordCompanionPercept({
|
|
38
|
+
modality: 'screen',
|
|
39
|
+
source: 'sensory_screen_reaction',
|
|
40
|
+
summary: `Screen changed${analysis.description ? ` → ${analysis.description}` : ''}`,
|
|
41
|
+
confidence: 0.85,
|
|
42
|
+
payload: { score, description: analysis.description },
|
|
43
|
+
tags: ['screen', 'change'],
|
|
44
|
+
}, options.cwd ? { cwd: options.cwd } : {});
|
|
45
|
+
logger.info(`[screen] change recorded${analysis.description ? ` → ${analysis.description}` : ''}`);
|
|
46
|
+
}
|
|
47
|
+
catch (err) {
|
|
48
|
+
logger.warn(`[screen] reaction failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
49
|
+
}
|
|
50
|
+
finally {
|
|
51
|
+
inFlight = false;
|
|
52
|
+
}
|
|
53
|
+
})();
|
|
54
|
+
});
|
|
55
|
+
return () => {
|
|
56
|
+
bus.off(id);
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
//# sourceMappingURL=screen-reaction.js.map
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Sensory bridge — the ingress for the Rust `buddy-sense` nervous-system daemon.
|
|
3
|
+
*
|
|
4
|
+
* A loopback-only WebSocket server that receives `SensoryEvent` JSON frames from
|
|
5
|
+
* the daemon and re-emits them onto Code Buddy's internal event bus as
|
|
6
|
+
* `sensory:perception` events. Reactions (reactions.ts) subscribe to the bus.
|
|
7
|
+
*
|
|
8
|
+
* Opt-in via `CODEBUDDY_SENSORY=true` (wired in `buddy server`). Loopback-only +
|
|
9
|
+
* optional shared token. Never throws on a malformed frame.
|
|
10
|
+
*
|
|
11
|
+
* @module sensory/sensory-bridge
|
|
12
|
+
*/
|
|
13
|
+
export interface SensoryBridgeOptions {
|
|
14
|
+
host?: string;
|
|
15
|
+
port?: number;
|
|
16
|
+
/** If set, frames must carry a matching `token` field. */
|
|
17
|
+
token?: string;
|
|
18
|
+
}
|
|
19
|
+
export interface SensoryBridgeHandle {
|
|
20
|
+
port: number;
|
|
21
|
+
close(): Promise<void>;
|
|
22
|
+
}
|
|
23
|
+
export declare function startSensoryBridge(options?: SensoryBridgeOptions): SensoryBridgeHandle;
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Sensory bridge — the ingress for the Rust `buddy-sense` nervous-system daemon.
|
|
3
|
+
*
|
|
4
|
+
* A loopback-only WebSocket server that receives `SensoryEvent` JSON frames from
|
|
5
|
+
* the daemon and re-emits them onto Code Buddy's internal event bus as
|
|
6
|
+
* `sensory:perception` events. Reactions (reactions.ts) subscribe to the bus.
|
|
7
|
+
*
|
|
8
|
+
* Opt-in via `CODEBUDDY_SENSORY=true` (wired in `buddy server`). Loopback-only +
|
|
9
|
+
* optional shared token. Never throws on a malformed frame.
|
|
10
|
+
*
|
|
11
|
+
* @module sensory/sensory-bridge
|
|
12
|
+
*/
|
|
13
|
+
import { WebSocketServer } from 'ws';
|
|
14
|
+
import { getGlobalEventBus } from '../events/event-bus.js';
|
|
15
|
+
import { logger } from '../utils/logger.js';
|
|
16
|
+
/** The trust boundary: only known modalities pass; kinds are sanitized to a safe
|
|
17
|
+
* shape (no control chars / newlines, ≤64 chars) so a crafted local frame can't
|
|
18
|
+
* inject content downstream (e.g. into CODEBUDDY_MEMORY.md via dreaming). */
|
|
19
|
+
const KNOWN_MODALITIES = new Set(['audio', 'vision', 'screen', 'vital', 'ui']);
|
|
20
|
+
function sanitizeKind(kind) {
|
|
21
|
+
return kind.replace(/[^a-zA-Z0-9_:.-]/g, '').slice(0, 64);
|
|
22
|
+
}
|
|
23
|
+
export function startSensoryBridge(options = {}) {
|
|
24
|
+
const host = options.host ?? '127.0.0.1';
|
|
25
|
+
const port = options.port ?? Number(process.env.CODEBUDDY_SENSORY_PORT ?? 8129);
|
|
26
|
+
const token = options.token ?? process.env.CODEBUDDY_SENSORY_TOKEN;
|
|
27
|
+
const bus = getGlobalEventBus();
|
|
28
|
+
const wss = new WebSocketServer({ host, port });
|
|
29
|
+
wss.on('connection', (ws, req) => {
|
|
30
|
+
// Reject cross-origin (CSWSH): the Rust daemon sends NO Origin header; a
|
|
31
|
+
// browser page / extension always does. This stops a local web origin from
|
|
32
|
+
// injecting frames (and, with the camera reaction on, triggering the webcam).
|
|
33
|
+
if (req.headers.origin) {
|
|
34
|
+
ws.close();
|
|
35
|
+
return;
|
|
36
|
+
}
|
|
37
|
+
// Loopback-only: exact match, tolerating IPv4-mapped IPv6 (::ffff:127.0.0.1).
|
|
38
|
+
const remote = (req.socket.remoteAddress ?? '').replace(/^::ffff:/, '');
|
|
39
|
+
if (remote !== '127.0.0.1' && remote !== '::1') {
|
|
40
|
+
ws.close();
|
|
41
|
+
return;
|
|
42
|
+
}
|
|
43
|
+
logger.info('[sensory] daemon connected');
|
|
44
|
+
ws.on('message', (data) => {
|
|
45
|
+
let frame;
|
|
46
|
+
try {
|
|
47
|
+
frame = JSON.parse(String(data));
|
|
48
|
+
}
|
|
49
|
+
catch {
|
|
50
|
+
return; // ignore malformed
|
|
51
|
+
}
|
|
52
|
+
if (token && frame.token !== token)
|
|
53
|
+
return;
|
|
54
|
+
if (!frame.modality || !frame.kind)
|
|
55
|
+
return;
|
|
56
|
+
// Validate the string fields at the trust boundary (not just the numbers):
|
|
57
|
+
// unknown modality is dropped, kind is sanitized — blocks memory injection.
|
|
58
|
+
if (!KNOWN_MODALITIES.has(frame.modality))
|
|
59
|
+
return;
|
|
60
|
+
const kind = sanitizeKind(frame.kind);
|
|
61
|
+
if (!kind)
|
|
62
|
+
return;
|
|
63
|
+
// Validate/clamp numeric fields so a malformed frame can't poison downstream.
|
|
64
|
+
const salience = Number.isFinite(frame.salience) ? Math.max(0, Math.min(255, Math.round(frame.salience))) : 0;
|
|
65
|
+
const tsMs = Number.isFinite(frame.ts_ms) && frame.ts_ms >= 0 ? frame.ts_ms : undefined;
|
|
66
|
+
bus.emit('sensory:perception', {
|
|
67
|
+
source: 'buddy-sense',
|
|
68
|
+
metadata: {
|
|
69
|
+
modality: frame.modality,
|
|
70
|
+
kind,
|
|
71
|
+
salience,
|
|
72
|
+
tsMs,
|
|
73
|
+
payload: frame.payload,
|
|
74
|
+
},
|
|
75
|
+
});
|
|
76
|
+
});
|
|
77
|
+
});
|
|
78
|
+
wss.on('error', (err) => logger.warn(`[sensory] bridge error: ${err instanceof Error ? err.message : String(err)}`));
|
|
79
|
+
logger.info(`[sensory] bridge listening on ws://${host}:${port}`);
|
|
80
|
+
return {
|
|
81
|
+
port,
|
|
82
|
+
close: () => new Promise((resolve) => wss.close(() => resolve())),
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
//# sourceMappingURL=sensory-bridge.js.map
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Short-term sensory memory — a bounded rolling buffer of recent perceptions
|
|
3
|
+
* (the nervous system's working memory). Reactions push every perception here;
|
|
4
|
+
* the dreaming consolidator drains + summarizes it into long-term memory every N
|
|
5
|
+
* heartbeats (like OpenClaw's dreaming: short-term recall → consolidated store).
|
|
6
|
+
*
|
|
7
|
+
* @module sensory/sensory-memory
|
|
8
|
+
*/
|
|
9
|
+
import type { Perception } from './reactions.js';
|
|
10
|
+
export declare class SensoryMemory {
|
|
11
|
+
private readonly cap;
|
|
12
|
+
private buf;
|
|
13
|
+
constructor(cap?: number);
|
|
14
|
+
push(p: Perception): void;
|
|
15
|
+
snapshot(): Perception[];
|
|
16
|
+
/** Take everything and clear (consolidation consumes the window). */
|
|
17
|
+
drain(): Perception[];
|
|
18
|
+
size(): number;
|
|
19
|
+
}
|
|
20
|
+
export declare function getSensoryMemory(): SensoryMemory;
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Short-term sensory memory — a bounded rolling buffer of recent perceptions
|
|
3
|
+
* (the nervous system's working memory). Reactions push every perception here;
|
|
4
|
+
* the dreaming consolidator drains + summarizes it into long-term memory every N
|
|
5
|
+
* heartbeats (like OpenClaw's dreaming: short-term recall → consolidated store).
|
|
6
|
+
*
|
|
7
|
+
* @module sensory/sensory-memory
|
|
8
|
+
*/
|
|
9
|
+
export class SensoryMemory {
|
|
10
|
+
cap;
|
|
11
|
+
buf = [];
|
|
12
|
+
constructor(cap = 1000) {
|
|
13
|
+
this.cap = cap;
|
|
14
|
+
}
|
|
15
|
+
push(p) {
|
|
16
|
+
this.buf.push(p);
|
|
17
|
+
if (this.buf.length > this.cap)
|
|
18
|
+
this.buf.shift();
|
|
19
|
+
}
|
|
20
|
+
snapshot() {
|
|
21
|
+
return [...this.buf];
|
|
22
|
+
}
|
|
23
|
+
/** Take everything and clear (consolidation consumes the window). */
|
|
24
|
+
drain() {
|
|
25
|
+
const out = this.buf;
|
|
26
|
+
this.buf = [];
|
|
27
|
+
return out;
|
|
28
|
+
}
|
|
29
|
+
size() {
|
|
30
|
+
return this.buf.length;
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
let singleton;
|
|
34
|
+
export function getSensoryMemory() {
|
|
35
|
+
if (!singleton)
|
|
36
|
+
singleton = new SensoryMemory();
|
|
37
|
+
return singleton;
|
|
38
|
+
}
|
|
39
|
+
//# sourceMappingURL=sensory-memory.js.map
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Speech reaction — closes the perception→cognition loop. On a `speech_end` event
|
|
3
|
+
* that carries the source WAV (the daemon tags it), transcribe the utterance (STT)
|
|
4
|
+
* and record what was heard as a `hearing` percept; an optional `onHeard` hook can
|
|
5
|
+
* drive further action (a turn, a command). DEBOUNCED (one transcription per
|
|
6
|
+
* utterance — the energy VAD over-segments), opt-in (`CODEBUDDY_SENSORY_SPEECH=true`),
|
|
7
|
+
* injectable transcriber, never-throws.
|
|
8
|
+
*
|
|
9
|
+
* @module sensory/speech-reaction
|
|
10
|
+
*/
|
|
11
|
+
export type Transcriber = (wav: string) => Promise<string>;
|
|
12
|
+
export interface SpeechReactionOptions {
|
|
13
|
+
/** Injectable STT (tests / custom). Default: faster-whisper via python ($0). */
|
|
14
|
+
transcriber?: Transcriber;
|
|
15
|
+
debounceMs?: number;
|
|
16
|
+
cwd?: string;
|
|
17
|
+
now?: () => number;
|
|
18
|
+
/** Action hook for the transcript (e.g. trigger an agent turn). */
|
|
19
|
+
onHeard?: (text: string) => void | Promise<void>;
|
|
20
|
+
}
|
|
21
|
+
export declare function wireSpeechReaction(options?: SpeechReactionOptions): () => void;
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Speech reaction — closes the perception→cognition loop. On a `speech_end` event
|
|
3
|
+
* that carries the source WAV (the daemon tags it), transcribe the utterance (STT)
|
|
4
|
+
* and record what was heard as a `hearing` percept; an optional `onHeard` hook can
|
|
5
|
+
* drive further action (a turn, a command). DEBOUNCED (one transcription per
|
|
6
|
+
* utterance — the energy VAD over-segments), opt-in (`CODEBUDDY_SENSORY_SPEECH=true`),
|
|
7
|
+
* injectable transcriber, never-throws.
|
|
8
|
+
*
|
|
9
|
+
* @module sensory/speech-reaction
|
|
10
|
+
*/
|
|
11
|
+
import { getGlobalEventBus } from '../events/event-bus.js';
|
|
12
|
+
import { logger } from '../utils/logger.js';
|
|
13
|
+
import { perceptionOf } from './reactions.js';
|
|
14
|
+
/** Default transcriber: local faster-whisper (base), best-effort, $0. */
|
|
15
|
+
async function defaultTranscribe(wav) {
|
|
16
|
+
const { spawn } = await import('child_process');
|
|
17
|
+
const model = process.env.CODEBUDDY_SPEECH_MODEL ?? 'base';
|
|
18
|
+
const py = [
|
|
19
|
+
'import sys',
|
|
20
|
+
'from faster_whisper import WhisperModel',
|
|
21
|
+
`m = WhisperModel(${JSON.stringify(model)}, device='cpu', compute_type='int8')`,
|
|
22
|
+
'segs, _ = m.transcribe(sys.argv[1])',
|
|
23
|
+
"print(' '.join(s.text for s in segs).strip())",
|
|
24
|
+
].join('\n');
|
|
25
|
+
return new Promise((resolve) => {
|
|
26
|
+
const proc = spawn('python3', ['-c', py, wav], { stdio: ['ignore', 'pipe', 'ignore'] });
|
|
27
|
+
let out = '';
|
|
28
|
+
proc.stdout.on('data', (d) => (out += String(d)));
|
|
29
|
+
proc.on('close', () => resolve(out.trim()));
|
|
30
|
+
proc.on('error', () => resolve(''));
|
|
31
|
+
});
|
|
32
|
+
}
|
|
33
|
+
export function wireSpeechReaction(options = {}) {
|
|
34
|
+
const bus = getGlobalEventBus();
|
|
35
|
+
const debounceMs = options.debounceMs ?? Number(process.env.CODEBUDDY_SPEECH_DEBOUNCE_MS ?? 4000);
|
|
36
|
+
const now = options.now ?? (() => Date.now());
|
|
37
|
+
const transcribe = options.transcriber ?? defaultTranscribe;
|
|
38
|
+
let lastAt = Number.NEGATIVE_INFINITY;
|
|
39
|
+
let inFlight = false;
|
|
40
|
+
const id = bus.on('sensory:perception', (evt) => {
|
|
41
|
+
const p = perceptionOf(evt);
|
|
42
|
+
if (p.modality !== 'audio' || p.kind !== 'speech_end')
|
|
43
|
+
return;
|
|
44
|
+
const wav = p.payload?.wav;
|
|
45
|
+
if (!wav)
|
|
46
|
+
return; // no audio to transcribe (e.g. a live-mic path not yet wired)
|
|
47
|
+
const t = now();
|
|
48
|
+
if (t - lastAt < debounceMs)
|
|
49
|
+
return; // one transcription per utterance
|
|
50
|
+
if (inFlight)
|
|
51
|
+
return; // a prior STT (faster-whisper, seconds) is still running
|
|
52
|
+
lastAt = t;
|
|
53
|
+
inFlight = true;
|
|
54
|
+
void (async () => {
|
|
55
|
+
try {
|
|
56
|
+
const text = await transcribe(wav);
|
|
57
|
+
if (!text)
|
|
58
|
+
return;
|
|
59
|
+
const { recordCompanionPercept } = await import('../companion/percepts.js');
|
|
60
|
+
await recordCompanionPercept({
|
|
61
|
+
modality: 'hearing',
|
|
62
|
+
source: 'sensory_speech_reaction',
|
|
63
|
+
summary: `Heard: ${text}`,
|
|
64
|
+
confidence: 0.8,
|
|
65
|
+
payload: { text, wav },
|
|
66
|
+
tags: ['speech', 'stt'],
|
|
67
|
+
}, options.cwd ? { cwd: options.cwd } : {});
|
|
68
|
+
logger.info(`[speech] heard → ${text}`);
|
|
69
|
+
await options.onHeard?.(text);
|
|
70
|
+
}
|
|
71
|
+
catch (err) {
|
|
72
|
+
logger.warn(`[speech] reaction failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
73
|
+
}
|
|
74
|
+
finally {
|
|
75
|
+
inFlight = false;
|
|
76
|
+
}
|
|
77
|
+
})();
|
|
78
|
+
});
|
|
79
|
+
return () => {
|
|
80
|
+
bus.off(id);
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
//# sourceMappingURL=speech-reaction.js.map
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Vision reaction — on a `vision/motion` event from the nervous-system daemon,
|
|
3
|
+
* run `camera_analyze` (capture a frame + describe it with a local gemma vision
|
|
4
|
+
* model) and record a companion percept. DEBOUNCED (gemma is heavy: 3–10s warm),
|
|
5
|
+
* opt-in (`CODEBUDDY_SENSORY_CAMERA=true`), best-effort, never-throws.
|
|
6
|
+
*
|
|
7
|
+
* @module sensory/vision-reaction
|
|
8
|
+
*/
|
|
9
|
+
export interface VisionAnalysis {
|
|
10
|
+
success: boolean;
|
|
11
|
+
description?: string;
|
|
12
|
+
imagePath?: string;
|
|
13
|
+
}
|
|
14
|
+
export interface VisionAnalyzer {
|
|
15
|
+
analyze(prompt: string): Promise<VisionAnalysis>;
|
|
16
|
+
}
|
|
17
|
+
export interface VisionReactionOptions {
|
|
18
|
+
/** Injectable analyzer (tests / custom). Defaults to camera_analyze (local gemma). */
|
|
19
|
+
analyzer?: VisionAnalyzer;
|
|
20
|
+
debounceMs?: number;
|
|
21
|
+
cwd?: string;
|
|
22
|
+
now?: () => number;
|
|
23
|
+
}
|
|
24
|
+
/** Security invariant (pure + testable): the camera reaction may only be wired
|
|
25
|
+
* when explicitly enabled AND a shared token is set — a crafted local frame can
|
|
26
|
+
* trigger the webcam, so an unauthenticated bridge must not be able to. */
|
|
27
|
+
export declare function shouldWireVisionReaction(env: {
|
|
28
|
+
camera?: string;
|
|
29
|
+
token?: string;
|
|
30
|
+
}): boolean;
|
|
31
|
+
export declare function wireVisionReaction(options?: VisionReactionOptions): () => void;
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Vision reaction — on a `vision/motion` event from the nervous-system daemon,
|
|
3
|
+
* run `camera_analyze` (capture a frame + describe it with a local gemma vision
|
|
4
|
+
* model) and record a companion percept. DEBOUNCED (gemma is heavy: 3–10s warm),
|
|
5
|
+
* opt-in (`CODEBUDDY_SENSORY_CAMERA=true`), best-effort, never-throws.
|
|
6
|
+
*
|
|
7
|
+
* @module sensory/vision-reaction
|
|
8
|
+
*/
|
|
9
|
+
import { getGlobalEventBus } from '../events/event-bus.js';
|
|
10
|
+
import { logger } from '../utils/logger.js';
|
|
11
|
+
import { perceptionOf } from './reactions.js';
|
|
12
|
+
/** Default analyzer: Code Buddy's camera_analyze tool (ffmpeg capture + local gemma). */
|
|
13
|
+
async function defaultAnalyze(prompt) {
|
|
14
|
+
const { CameraAnalyzeTool } = await import('../tools/registry/vision-tools.js');
|
|
15
|
+
const tool = new CameraAnalyzeTool({});
|
|
16
|
+
const result = await tool.execute({ prompt });
|
|
17
|
+
const data = result.data;
|
|
18
|
+
return { success: result.success, description: data?.description ?? result.output, imagePath: data?.imagePath };
|
|
19
|
+
}
|
|
20
|
+
/** Security invariant (pure + testable): the camera reaction may only be wired
|
|
21
|
+
* when explicitly enabled AND a shared token is set — a crafted local frame can
|
|
22
|
+
* trigger the webcam, so an unauthenticated bridge must not be able to. */
|
|
23
|
+
export function shouldWireVisionReaction(env) {
|
|
24
|
+
return env.camera === 'true' && Boolean(env.token);
|
|
25
|
+
}
|
|
26
|
+
export function wireVisionReaction(options = {}) {
|
|
27
|
+
const bus = getGlobalEventBus();
|
|
28
|
+
const debounceMs = options.debounceMs ?? Number(process.env.CODEBUDDY_VISION_DEBOUNCE_MS ?? 8000);
|
|
29
|
+
const now = options.now ?? (() => Date.now());
|
|
30
|
+
const analyzer = options.analyzer ?? { analyze: defaultAnalyze };
|
|
31
|
+
let lastAt = Number.NEGATIVE_INFINITY;
|
|
32
|
+
let inFlight = false;
|
|
33
|
+
const id = bus.on('sensory:perception', (evt) => {
|
|
34
|
+
const p = perceptionOf(evt);
|
|
35
|
+
if (p.modality !== 'vision' || p.kind !== 'motion')
|
|
36
|
+
return;
|
|
37
|
+
const t = now();
|
|
38
|
+
if (t - lastAt < debounceMs) {
|
|
39
|
+
logger.info('[vision] motion (debounced — analysis throttled)');
|
|
40
|
+
return;
|
|
41
|
+
}
|
|
42
|
+
if (inFlight)
|
|
43
|
+
return; // a prior analyze() (ffmpeg+gemma, 3–10s) is still running
|
|
44
|
+
lastAt = t;
|
|
45
|
+
inFlight = true;
|
|
46
|
+
void (async () => {
|
|
47
|
+
try {
|
|
48
|
+
const res = await analyzer.analyze('Motion detected — describe the scene in one sentence.');
|
|
49
|
+
if (!res.success)
|
|
50
|
+
return;
|
|
51
|
+
const { recordCompanionPercept } = await import('../companion/percepts.js');
|
|
52
|
+
await recordCompanionPercept({
|
|
53
|
+
modality: 'vision',
|
|
54
|
+
source: 'sensory_motion_reaction',
|
|
55
|
+
summary: `Motion → ${res.description ?? '(no description)'}`,
|
|
56
|
+
confidence: 0.9,
|
|
57
|
+
payload: { description: res.description, imagePath: res.imagePath },
|
|
58
|
+
tags: ['motion', 'camera', 'vision'],
|
|
59
|
+
}, options.cwd ? { cwd: options.cwd } : {});
|
|
60
|
+
logger.info(`[vision] motion analyzed → ${res.description ?? '(no description)'}`);
|
|
61
|
+
}
|
|
62
|
+
catch (err) {
|
|
63
|
+
logger.warn(`[vision] reaction failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
64
|
+
}
|
|
65
|
+
finally {
|
|
66
|
+
inFlight = false;
|
|
67
|
+
}
|
|
68
|
+
})();
|
|
69
|
+
});
|
|
70
|
+
return () => {
|
|
71
|
+
bus.off(id);
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
//# sourceMappingURL=vision-reaction.js.map
|