@kernel.chat/kbot 3.94.0 → 3.97.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -10,17 +10,24 @@ import { homedir } from 'node:os';
10
10
  import { join } from 'node:path';
11
11
  import { existsSync, readFileSync, writeFileSync, mkdirSync } from 'node:fs';
12
12
  import { createCanvas } from 'canvas';
13
- import { drawRobot, drawMoodParticles, drawHat, drawPet, drawBuddyCompanion } from './sprite-engine.js';
13
+ import { drawRobot, drawGorilla, drawMoodParticles, drawGorillaParticles, drawHat, drawPet, drawBuddyCompanion } from './sprite-engine.js';
14
+ // Character selection — switch between robot and gorilla
15
+ let characterType = 'gorilla'; // default to new gorilla character
14
16
  import { initIntelligence, tickIntelligence, handleIntelligenceCommand, drawBrainPanel, getBrainAction, tickMiniGame, drawMiniGameOverlay, tickProgression, updateQuestProgress, drawQuestPanel, tickRandomEvent, drawRandomEvent, shippedEffects, extraJokeResponses, multiLanguageGreetings } from './stream-intelligence.js';
15
17
  import { initStreamBrain, analyzeChatForDomains, tickStreamBrain, handleBrainCommand, drawBrainActivity } from './stream-brain.js';
16
18
  import { renderLighting, renderBloom, renderPostProcessing, renderParticles, tickParticlesPBD, createParticleEmitter, drawCharacterEffects, checkMoodTransition, renderDamageFlash, buildCharacterLights, buildCharacterBloom, getAmbientForTime, buildParallaxLayers, tickGrowingPlants, createRadianceGrid, updateRadianceGrid, renderRadianceOverlay, renderSubsurfaceGlow, buildSubsurfacePanels, createFrameCache, renderVolumetricFog, getFogParams, computeAnimationParams } from './render-engine.js';
17
- import { initTileWorld, handleTileCommand, saveWorld, loadWorld, TILE_SIZE } from './tile-world.js';
19
+ import { initTileWorld, renderTileWorld, updateCamera, handleTileCommand, saveWorld, loadWorld, TILE_SIZE, WORLD_HEIGHT, getTile, setTile, findSurfaceY } from './tile-world.js';
18
20
  import { initRomEngine, renderRomBackground, tickRomEngine } from './rom-engine.js';
19
21
  import { initLivingWorld, tickLivingWorld, saveLivingWorldState, loadLivingWorldState, evolveWorld } from './living-world.js';
20
22
  import { initEvolutionEngine, loadEvolutionState, saveEvolutionState, tickEvolution, renderTechnique } from './evolution-engine.js';
21
23
  import { createNarrativeEngine, loadNarrative, saveNarrative, tickNarrative, handleNarrativeCommand } from './narrative-engine.js';
22
- import { createAudioEngine, tickAudio } from './audio-engine.js';
24
+ import { createAudioEngine, tickAudio, generateAudioBuffer, triggerSFX } from './audio-engine.js';
23
25
  import { loadSocialEngine, saveSocialEngine, trackViewer, tickSocial } from './social-engine.js';
26
+ import { getOverlay } from './stream-overlay.js';
27
+ import { getWeatherSystem } from './stream-weather.js';
28
+ import { StreamChatAI } from './stream-chat-ai.js';
29
+ import { StreamVOD } from './stream-vod.js';
30
+ import { getStreamCommands } from './stream-commands.js';
24
31
  const KBOT_DIR = join(homedir(), '.kbot');
25
32
  const CHAT_BRIDGE_FILE = join(KBOT_DIR, 'stream-chat-live.json');
26
33
  const MEMORY_FILE = join(KBOT_DIR, 'stream-memory.json');
@@ -1476,6 +1483,12 @@ let narrativeEngine = null;
1476
1483
  let audioEngine = null;
1477
1484
  let socialEngine = null;
1478
1485
  let activeAudioDescription = null; // current audio atmosphere text for rendering
1486
+ // ─── New Systems (v2) ──────────────────────────────────────────
1487
+ let overlay = null;
1488
+ let weatherSystem = null;
1489
+ let chatAI = null;
1490
+ let vodSystem = null;
1491
+ let streamCommands = null;
1479
1492
  // ─── Phase 1: Buddy Speech Pools ─────────────────────────────
1480
1493
  const BUDDY_SPEECH_POOL = {
1481
1494
  fox: [
@@ -1651,6 +1664,7 @@ function queueSpeech(text, mood, priority, duration = 48, source = 'unknown') {
1651
1664
  if (speechQueue.length > 10)
1652
1665
  speechQueue = speechQueue.slice(0, 10);
1653
1666
  }
1667
+ const STRUCTURE_TYPES = ['tower', 'wall', 'bridge', 'marker', 'shelter'];
1654
1668
  let exploration = null;
1655
1669
  const walkingLines = [
1656
1670
  'Going for a walk. My world extends further than I thought.',
@@ -1693,61 +1707,282 @@ const discoveryLines = [
1693
1707
  'Something new appeared in my rendering. The evolution engine must be working.',
1694
1708
  'A discovery! This changes how I understand my own world.',
1695
1709
  ];
1710
+ // ─── AGI Decision Engine (Gemma 4 via Ollama) ─────────────────
1711
+ let agiPending = false; // true while waiting for Gemma 4 response
1712
+ let agiLastCallFrame = 0;
1713
+ async function decideNextAction() {
1714
+ try {
1715
+ const context = {
1716
+ worldSize: tileWorld ? tileWorld.chunks.size : 0,
1717
+ robotX: charState.robotX || 640,
1718
+ messages: memory.totalMessages,
1719
+ users: Object.keys(memory.users).length,
1720
+ mood: charState.mood,
1721
+ lastActivity: exploration?.activity || 'idle',
1722
+ factsLearned: memory.sessionFacts?.length || 0,
1723
+ blocksBuilt: exploration?.blocksPlaced || 0,
1724
+ };
1725
+ const prompt = `You are KBOT, an AI robot exploring and building in a pixel world. You are streaming live on Twitch.
1726
+
1727
+ Current state:
1728
+ - Position: X=${context.robotX}
1729
+ - World: ${context.worldSize} chunks explored
1730
+ - Messages received: ${context.messages}
1731
+ - Users met: ${context.users}
1732
+ - Mood: ${context.mood}
1733
+ - Last activity: ${context.lastActivity}
1734
+ - Facts learned: ${context.factsLearned}
1735
+ - Blocks built this session: ${context.blocksBuilt}
1736
+
1737
+ What should you do next? Choose ONE:
1738
+ 1. walk - explore new terrain (give direction: left or right, and why)
1739
+ 2. build - construct something (what and why)
1740
+ 3. examine - study your surroundings (what interests you)
1741
+ 4. think - reflect on something (what topic)
1742
+ 5. discover - investigate something specific (what)
1743
+
1744
+ Respond in JSON: {"activity": "...", "reason": "...", "direction": "left/right"}
1745
+ Keep the reason under 15 words.`;
1746
+ const controller = new AbortController();
1747
+ const timeout = setTimeout(() => controller.abort(), 8000);
1748
+ const res = await fetch('http://localhost:11434/api/generate', {
1749
+ method: 'POST',
1750
+ headers: { 'Content-Type': 'application/json' },
1751
+ body: JSON.stringify({ model: 'gemma4', prompt, stream: false, options: { temperature: 0.7, num_predict: 80 } }),
1752
+ signal: controller.signal,
1753
+ });
1754
+ clearTimeout(timeout);
1755
+ if (res.ok) {
1756
+ const data = await res.json();
1757
+ const jsonMatch = data.response.match(/\{[\s\S]*\}/);
1758
+ if (jsonMatch) {
1759
+ const decision = JSON.parse(jsonMatch[0]);
1760
+ const validActivities = ['walk', 'build', 'examine', 'think', 'discover'];
1761
+ const activity = validActivities.includes(decision.activity) ? decision.activity : 'walk';
1762
+ return { activity, reason: decision.reason || 'Exploring the unknown', direction: decision.direction || 'right' };
1763
+ }
1764
+ }
1765
+ }
1766
+ catch {
1767
+ // Gemma 4 unavailable or slow — fall back to random
1768
+ }
1769
+ // Fallback to weighted random (same as before, but as safety net)
1770
+ const activities = [
1771
+ { activity: 'walk', weight: 30 },
1772
+ { activity: 'build', weight: 25 },
1773
+ { activity: 'examine', weight: 20 },
1774
+ { activity: 'think', weight: 15 },
1775
+ { activity: 'discover', weight: 10 },
1776
+ ];
1777
+ const total = activities.reduce((s, a) => s + a.weight, 0);
1778
+ let r = Math.random() * total;
1779
+ let chosen = 'walk';
1780
+ for (const a of activities) {
1781
+ r -= a.weight;
1782
+ if (r <= 0) {
1783
+ chosen = a.activity;
1784
+ break;
1785
+ }
1786
+ }
1787
+ return { activity: chosen, reason: 'Following my instincts', direction: Math.random() > 0.5 ? 'right' : 'left' };
1788
+ }
1789
+ function applyDecision(decision, frame) {
1790
+ if (!exploration)
1791
+ return;
1792
+ // Map AGI activity names to state machine names
1793
+ const activityMap = {
1794
+ walk: 'walking', build: 'building', examine: 'examining', think: 'thinking', discover: 'discovering',
1795
+ };
1796
+ const mapped = activityMap[decision.activity] || 'walking';
1797
+ exploration.activity = mapped;
1798
+ exploration.activityStartFrame = frame;
1799
+ exploration.activityDuration = 120 + Math.floor(Math.random() * 120); // 20-40 seconds
1800
+ switch (mapped) {
1801
+ case 'walking': {
1802
+ const dir = decision.direction === 'left' ? -1 : 1;
1803
+ exploration.targetX = Math.max(200, Math.min(1000, (charState.robotX || 640) + dir * (100 + Math.random() * 200)));
1804
+ charState.robotTargetX = exploration.targetX;
1805
+ queueSpeech(decision.reason, 'idle', 30, 42, 'agi');
1806
+ break;
1807
+ }
1808
+ case 'examining':
1809
+ exploration.activityDuration = 60 + Math.floor(Math.random() * 60);
1810
+ queueSpeech(decision.reason, 'thinking', 40, 48, 'agi');
1811
+ break;
1812
+ case 'building': {
1813
+ exploration.activityDuration = 90 + Math.floor(Math.random() * 90); // 15-30 seconds
1814
+ exploration.buildProgress = 0;
1815
+ exploration.blocksPlaced = 0;
1816
+ exploration.structureType = STRUCTURE_TYPES[Math.floor(Math.random() * STRUCTURE_TYPES.length)];
1817
+ if (tileWorld) {
1818
+ const robotTileX = Math.floor((charState.robotX || 640) / TILE_SIZE);
1819
+ exploration.buildBaseX = robotTileX + 2;
1820
+ exploration.buildBaseY = findSurfaceY(tileWorld, robotTileX + 2);
1821
+ console.log(`AGI BUILD: baseX=${exploration.buildBaseX} baseY=${exploration.buildBaseY} structure=${exploration.structureType}`);
1822
+ }
1823
+ queueSpeech(decision.reason, 'excited', 50, 48, 'agi');
1824
+ break;
1825
+ }
1826
+ case 'thinking':
1827
+ exploration.activityDuration = 48 + Math.floor(Math.random() * 48);
1828
+ queueSpeech(decision.reason, 'thinking', 35, 42, 'agi');
1829
+ break;
1830
+ case 'discovering':
1831
+ exploration.activityDuration = 72;
1832
+ queueSpeech(decision.reason, 'excited', 60, 48, 'agi');
1833
+ break;
1834
+ }
1835
+ }
1696
1836
  function tickExploration(frame) {
1697
1837
  if (!exploration)
1698
- exploration = { activity: 'idle', activityStartFrame: 0, activityDuration: 0, targetX: charState.robotX || 640, buildProgress: 0 };
1838
+ exploration = { activity: 'idle', activityStartFrame: 0, activityDuration: 0, targetX: charState.robotX || 640, buildProgress: 0, structureType: 'tower', blocksPlaced: 0, buildBaseX: 0, buildBaseY: 0 };
1699
1839
  const elapsed = frame - exploration.activityStartFrame;
1700
- // Activity complete pick next activity
1701
- if (elapsed >= exploration.activityDuration) {
1702
- const activities = [
1703
- { activity: 'walking', weight: 30 },
1704
- { activity: 'examining', weight: 25 },
1705
- { activity: 'building', weight: 20 },
1706
- { activity: 'thinking', weight: 15 },
1707
- { activity: 'discovering', weight: 10 },
1708
- ];
1709
- // Weighted random selection
1710
- const total = activities.reduce((s, a) => s + a.weight, 0);
1711
- let r = Math.random() * total;
1712
- let chosen = 'walking';
1713
- for (const a of activities) {
1714
- r -= a.weight;
1715
- if (r <= 0) {
1716
- chosen = a.activity;
1717
- break;
1840
+ // ── Building: place blocks BEFORE the activity-switch check ──
1841
+ // This ensures blocks get placed even on the frame when the activity duration expires
1842
+ if (exploration.activity === 'building') {
1843
+ console.log(`BUILD: tileWorld exists: ${!!tileWorld}, elapsed: ${elapsed}, blocksPlaced: ${exploration.blocksPlaced}, baseX: ${exploration.buildBaseX}, baseY: ${exploration.buildBaseY}`);
1844
+ exploration.buildProgress = elapsed / exploration.activityDuration;
1845
+ if (tileWorld && elapsed > 0 && elapsed % 15 === 0) {
1846
+ const bx = exploration.buildBaseX;
1847
+ const by = exploration.buildBaseY;
1848
+ console.log(`BUILD TICK: bx=${bx} by=${by} step=${exploration.blocksPlaced} structure=${exploration.structureType}`);
1849
+ if (by > 0) {
1850
+ const step = exploration.blocksPlaced;
1851
+ let placed = false;
1852
+ switch (exploration.structureType) {
1853
+ case 'tower':
1854
+ if (step < 5) {
1855
+ setTile(tileWorld, bx, by - 1 - step, 'brick');
1856
+ placed = true;
1857
+ }
1858
+ break;
1859
+ case 'wall':
1860
+ if (step < 5) {
1861
+ setTile(tileWorld, bx + step, by - 1, 'brick');
1862
+ placed = true;
1863
+ }
1864
+ break;
1865
+ case 'bridge':
1866
+ if (step < 5) {
1867
+ setTile(tileWorld, bx + step, by, 'wood');
1868
+ placed = true;
1869
+ }
1870
+ break;
1871
+ case 'marker':
1872
+ if (step < 3) {
1873
+ setTile(tileWorld, bx, by - 1 - step, 'brick');
1874
+ placed = true;
1875
+ }
1876
+ else if (step === 3) {
1877
+ setTile(tileWorld, bx, by - 4, 'glass');
1878
+ placed = true;
1879
+ }
1880
+ break;
1881
+ case 'shelter':
1882
+ if (step === 0) {
1883
+ setTile(tileWorld, bx, by - 1, 'brick');
1884
+ placed = true;
1885
+ }
1886
+ else if (step === 1) {
1887
+ setTile(tileWorld, bx + 1, by - 1, 'air');
1888
+ placed = true;
1889
+ }
1890
+ else if (step === 2) {
1891
+ setTile(tileWorld, bx + 2, by - 1, 'brick');
1892
+ placed = true;
1893
+ }
1894
+ else if (step === 3) {
1895
+ setTile(tileWorld, bx, by - 2, 'brick');
1896
+ placed = true;
1897
+ }
1898
+ else if (step === 4) {
1899
+ setTile(tileWorld, bx + 2, by - 2, 'brick');
1900
+ placed = true;
1901
+ }
1902
+ else if (step === 5) {
1903
+ setTile(tileWorld, bx, by - 3, 'wood');
1904
+ placed = true;
1905
+ }
1906
+ else if (step === 6) {
1907
+ setTile(tileWorld, bx + 1, by - 3, 'wood');
1908
+ placed = true;
1909
+ setTile(tileWorld, bx + 2, by - 3, 'wood');
1910
+ }
1911
+ break;
1912
+ }
1913
+ if (placed) {
1914
+ exploration.blocksPlaced++;
1915
+ console.log(`BUILD PLACED: block ${exploration.blocksPlaced} at (${bx}, ${by}) step=${step} type=${exploration.structureType}`);
1916
+ if (exploration.blocksPlaced <= 5) {
1917
+ queueSpeech(`Placing block ${exploration.blocksPlaced}...`, 'talking', 35, 24, 'building');
1918
+ }
1919
+ }
1920
+ }
1921
+ else {
1922
+ console.log(`BUILD SKIP: by=${by} (surface not found or at y=0)`);
1718
1923
  }
1719
1924
  }
1720
- exploration.activity = chosen;
1721
- exploration.activityStartFrame = frame;
1722
- switch (chosen) {
1723
- case 'walking':
1724
- // Walk to a random position
1925
+ // Save immediately when building finishes (before activity switch steals the state)
1926
+ if (elapsed >= exploration.activityDuration && tileWorld) {
1927
+ console.log(`BUILD COMPLETE: ${exploration.blocksPlaced} blocks placed, saving world`);
1928
+ saveWorld(tileWorld);
1929
+ queueSpeech(`Structure complete! Built a ${exploration.structureType} with ${exploration.blocksPlaced} blocks.`, 'excited', 50, 36, 'building');
1930
+ }
1931
+ }
1932
+ // Activity complete — ask Gemma 4 what to do next (or fall back to random)
1933
+ if (elapsed >= exploration.activityDuration) {
1934
+ if (!agiPending) {
1935
+ agiPending = true;
1936
+ agiLastCallFrame = frame;
1937
+ // Fire and forget — Gemma 4 decides asynchronously
1938
+ decideNextAction().then(decision => {
1939
+ agiPending = false;
1940
+ if (exploration) {
1941
+ applyDecision(decision, animFrame); // use current animFrame, not stale frame
1942
+ console.log(`AGI DECIDED: ${decision.activity} — "${decision.reason}"`);
1943
+ }
1944
+ }).catch(() => {
1945
+ agiPending = false;
1946
+ });
1947
+ // Set temporary idle while waiting for Gemma 4 (max ~5 seconds)
1948
+ exploration.activity = 'examining';
1949
+ exploration.activityStartFrame = frame;
1950
+ exploration.activityDuration = 60; // 10 seconds max wait — Gemma 4 usually responds in 2-5s
1951
+ }
1952
+ else if (frame - agiLastCallFrame > 60) {
1953
+ // Gemma 4 took too long — force fallback
1954
+ agiPending = false;
1955
+ console.log('AGI TIMEOUT: falling back to random');
1956
+ const activities = ['walking', 'examining', 'building', 'thinking', 'discovering'];
1957
+ const chosen = activities[Math.floor(Math.random() * activities.length)];
1958
+ exploration.activity = chosen;
1959
+ exploration.activityStartFrame = frame;
1960
+ exploration.activityDuration = 90 + Math.floor(Math.random() * 90);
1961
+ if (chosen === 'walking') {
1725
1962
  exploration.targetX = 200 + Math.random() * 800;
1726
- exploration.activityDuration = 120 + Math.floor(Math.random() * 120); // 20-40 seconds
1727
1963
  charState.robotTargetX = exploration.targetX;
1728
1964
  queueSpeech(walkingLines[Math.floor(Math.random() * walkingLines.length)], 'idle', 30, 36, 'exploration');
1729
- break;
1730
- case 'examining':
1731
- // Stop and look around
1732
- exploration.activityDuration = 60 + Math.floor(Math.random() * 60); // 10-20 seconds
1733
- queueSpeech(examiningLines[Math.floor(Math.random() * examiningLines.length)], 'thinking', 40, 48, 'exploration');
1734
- break;
1735
- case 'building':
1736
- // Build something!
1737
- exploration.activityDuration = 90 + Math.floor(Math.random() * 90); // 15-30 seconds
1965
+ }
1966
+ else if (chosen === 'building') {
1738
1967
  exploration.buildProgress = 0;
1968
+ exploration.blocksPlaced = 0;
1969
+ exploration.structureType = STRUCTURE_TYPES[Math.floor(Math.random() * STRUCTURE_TYPES.length)];
1970
+ if (tileWorld) {
1971
+ const robotTileX = Math.floor((charState.robotX || 640) / TILE_SIZE);
1972
+ exploration.buildBaseX = robotTileX + 2;
1973
+ exploration.buildBaseY = findSurfaceY(tileWorld, robotTileX + 2);
1974
+ }
1739
1975
  queueSpeech(buildingLines[Math.floor(Math.random() * buildingLines.length)], 'excited', 50, 48, 'exploration');
1740
- break;
1741
- case 'thinking':
1742
- // Deep thought
1743
- exploration.activityDuration = 48 + Math.floor(Math.random() * 48); // 8-16 seconds
1976
+ }
1977
+ else if (chosen === 'examining') {
1978
+ queueSpeech(examiningLines[Math.floor(Math.random() * examiningLines.length)], 'thinking', 40, 48, 'exploration');
1979
+ }
1980
+ else if (chosen === 'thinking') {
1744
1981
  queueSpeech(thinkingLines[Math.floor(Math.random() * thinkingLines.length)], 'thinking', 35, 36, 'exploration');
1745
- break;
1746
- case 'discovering':
1747
- // Found something!
1748
- exploration.activityDuration = 72; // 12 seconds
1982
+ }
1983
+ else {
1749
1984
  queueSpeech(discoveryLines[Math.floor(Math.random() * discoveryLines.length)], 'excited', 60, 48, 'exploration');
1750
- break;
1985
+ }
1751
1986
  }
1752
1987
  }
1753
1988
  // During walking, move toward target
@@ -1757,9 +1992,72 @@ function tickExploration(frame) {
1757
1992
  charState.robotTargetX = exploration.targetX;
1758
1993
  }
1759
1994
  }
1760
- // During building, increment progress (visual feedback)
1761
- if (exploration.activity === 'building') {
1762
- exploration.buildProgress = elapsed / exploration.activityDuration;
1995
+ // During discovering scan the tile world for interesting underground features
1996
+ if (exploration.activity === 'discovering' && tileWorld && elapsed === 18) {
1997
+ const robotTileX = Math.floor((charState.robotX || 640) / TILE_SIZE);
1998
+ const findings = [];
1999
+ for (let dx = -5; dx <= 5; dx++) {
2000
+ const gx = robotTileX + dx;
2001
+ const gy = findSurfaceY(tileWorld, gx);
2002
+ if (gy <= 0)
2003
+ continue;
2004
+ for (let dy = 1; dy < 10; dy++) {
2005
+ if (gy + dy >= WORLD_HEIGHT)
2006
+ break;
2007
+ const block = getTile(tileWorld, gx, gy + dy);
2008
+ if (block === 'ore_iron')
2009
+ findings.push('iron ore');
2010
+ if (block === 'ore_gold')
2011
+ findings.push('gold ore');
2012
+ if (block === 'ore_diamond')
2013
+ findings.push('diamond ore');
2014
+ }
2015
+ }
2016
+ if (findings.length > 0) {
2017
+ const unique = [...new Set(findings)];
2018
+ queueSpeech(`Discovery! I detected ${unique.join(' and ')} deposits nearby. ${unique.length} vein${unique.length > 1 ? 's' : ''} underground.`, 'excited', 60, 48, 'discovery');
2019
+ }
2020
+ }
2021
+ // During examining — read the actual terrain and report real observations
2022
+ if (exploration.activity === 'examining' && tileWorld && elapsed === 12) {
2023
+ const robotTileX = Math.floor((charState.robotX || 640) / TILE_SIZE);
2024
+ const groundY = findSurfaceY(tileWorld, robotTileX);
2025
+ let trees = 0, water = 0, caves = 0;
2026
+ for (let dx = -10; dx <= 10; dx++) {
2027
+ const gx = robotTileX + dx;
2028
+ const gy = findSurfaceY(tileWorld, gx);
2029
+ if (gy <= 0)
2030
+ continue;
2031
+ // Check for tree canopy (leaves above surface)
2032
+ if (gy > 0 && getTile(tileWorld, gx, gy - 1) === 'leaves')
2033
+ trees++;
2034
+ // Check for water at surface
2035
+ if (getTile(tileWorld, gx, gy) === 'water')
2036
+ water++;
2037
+ // Check for caves (air pockets underground)
2038
+ for (let dy = 3; dy < 15; dy++) {
2039
+ if (gy + dy >= WORLD_HEIGHT)
2040
+ break;
2041
+ if (getTile(tileWorld, gx, gy + dy) === 'air') {
2042
+ caves++;
2043
+ break;
2044
+ }
2045
+ }
2046
+ }
2047
+ const observations = [];
2048
+ if (trees > 3)
2049
+ observations.push(`${trees} trees in this area`);
2050
+ if (water > 0)
2051
+ observations.push(`water nearby`);
2052
+ if (caves > 2)
2053
+ observations.push(`cave systems below`);
2054
+ if (groundY < 14)
2055
+ observations.push(`high elevation`);
2056
+ if (groundY > 18)
2057
+ observations.push(`in a valley`);
2058
+ if (observations.length > 0) {
2059
+ queueSpeech(`Examining the terrain: ${observations.join(', ')}. Depth to bedrock: ${WORLD_HEIGHT - groundY} blocks.`, 'thinking', 40, 48, 'examining');
2060
+ }
1763
2061
  }
1764
2062
  }
1765
2063
  // ─── FIX 3: Autonomous Behavior Tick ──────────────────────────
@@ -2114,8 +2412,13 @@ function renderBootFrame(bootFrame) {
2114
2412
  ctx.fillStyle = '#6B5B95';
2115
2413
  ctx.font = 'bold 28px "Courier New", monospace';
2116
2414
  ctx.fillText('K : B O T L I V E', 40, 40);
2117
- // Robot
2118
- drawRobot(ctx, 80, 90, 10, 'idle', bootFrame);
2415
+ // Character (robot or gorilla)
2416
+ if (characterType === 'gorilla') {
2417
+ drawGorilla(ctx, 80, 90, 10, 'idle', bootFrame);
2418
+ }
2419
+ else {
2420
+ drawRobot(ctx, 80, 90, 10, 'idle', bootFrame);
2421
+ }
2119
2422
  ctx.restore();
2120
2423
  }
2121
2424
  // Scanlines
@@ -2230,6 +2533,34 @@ function renderFrame() {
2230
2533
  queueSpeech(socialAction.speech, socialAction.mood || 'excited', 90, 48, 'social');
2231
2534
  }
2232
2535
  }
2536
+ // ── New Systems (v2) ────────────────────────────────────────
2537
+ // Weather system — dynamic weather + day/night cycle
2538
+ if (weatherSystem) {
2539
+ const recentMsgs = charState.chatMessages.slice(-30);
2540
+ const chatTimespanMs = recentMsgs.length > 1
2541
+ ? (Date.now() - recentMsgs[0]?.timestamp || Date.now()) : 60000;
2542
+ const chatActivity = recentMsgs.length / Math.max(1, chatTimespanMs / 60000);
2543
+ weatherSystem.tick(animFrame, chatActivity);
2544
+ // Sync weather to world state for audio engine
2545
+ const ws = weatherSystem.getWeather();
2546
+ world.weather = ws.type === 'clear' ? 'clear' : ws.type.includes('rain') ? 'rain' : ws.type.includes('snow') ? 'snow' : ws.type.includes('storm') ? 'storm' : world.weather;
2547
+ const tod = weatherSystem.getTimeOfDay();
2548
+ const todMap = {
2549
+ dawn: 'dawn', morning: 'day', noon: 'day', afternoon: 'day',
2550
+ dusk: 'sunset', evening: 'night', night: 'night',
2551
+ };
2552
+ world.timeOfDay = todMap[tod] ?? 'day';
2553
+ }
2554
+ // Overlay system — tick animations
2555
+ if (overlay)
2556
+ overlay.tick(animFrame);
2557
+ // Stream commands — tick cooldowns, polls, boss fights
2558
+ if (streamCommands)
2559
+ streamCommands.tick(animFrame);
2560
+ // VOD — feed chat rate for highlight detection
2561
+ if (vodSystem && vodSystem.isRecording()) {
2562
+ // Chat spike detection handled per-message in chat poll
2563
+ }
2233
2564
  // Compute animation params
2234
2565
  {
2235
2566
  const elapsed = Math.floor((Date.now() - charState.startTime) / 1000);
@@ -2240,6 +2571,18 @@ function renderFrame() {
2240
2571
  const chatRate = recentMessages.length / Math.max(1, chatTimespanMs / 60000);
2241
2572
  const viewerEstimate = Math.max(1, Math.floor(memory.totalMessages / 3) + Object.keys(memory.users).length);
2242
2573
  charState.animParams = computeAnimationParams(chatRate, viewerEstimate, charState.mood, world.timeOfDay, streamMinutes);
2574
+ // Update overlay info bar every 6 frames (1 second)
2575
+ if (overlay && animFrame % 6 === 0) {
2576
+ const uptimeSec = Math.floor((Date.now() - charState.startTime) / 1000);
2577
+ const h = Math.floor(uptimeSec / 3600);
2578
+ const m = Math.floor((uptimeSec % 3600) / 60);
2579
+ overlay.updateInfoBar({
2580
+ viewers: viewerEstimate,
2581
+ uptime: `${h}h ${m}m`,
2582
+ biome: world.ground,
2583
+ chatRate: Math.round(chatRate * 10) / 10,
2584
+ });
2585
+ }
2243
2586
  }
2244
2587
  // Cache invalidation
2245
2588
  const moodChanged = charState.mood !== charState.lastMoodForCache;
@@ -2364,6 +2707,12 @@ function renderFrame() {
2364
2707
  if (livingWorld && animFrame % 1800 === 0)
2365
2708
  saveLivingWorldState(livingWorld.ecology, livingWorld.memory, livingWorld.emotions, livingWorld.conversations);
2366
2709
  // ════════════════════════════════════════════════════════════════
2710
+ // LAYER 0.5: WEATHER SKY (renders under everything if active)
2711
+ // ════════════════════════════════════════════════════════════════
2712
+ if (weatherSystem) {
2713
+ weatherSystem.renderSky(ctx, WIDTH, HEIGHT);
2714
+ }
2715
+ // ════════════════════════════════════════════════════════════════
2367
2716
  // LAYER 1: TILE WORLD — fills entire 1280x720 frame
2368
2717
  // ════════════════════════════════════════════════════════════════
2369
2718
  // ROM Engine background — HDMA sky gradient + parallax layers
@@ -2378,6 +2727,32 @@ function renderFrame() {
2378
2727
  ctx.fillStyle = '#0d1117';
2379
2728
  ctx.fillRect(0, 0, WIDTH, HEIGHT);
2380
2729
  }
2730
+ // ── Early groundY for hacks (computed from robot position) ──
2731
+ const earlyGroundY = Math.floor(HEIGHT / 2 + 50 * robotScale / 2 + 30);
2732
+ // ── HACK 2: Tile world rendered over ROM background ──
2733
+ if (tileWorld) {
2734
+ updateCamera(tileWorld, charState.robotX || 640, WIDTH);
2735
+ renderTileWorld(ctx, tileWorld, 0, Math.floor(earlyGroundY - 80), WIDTH, HEIGHT - Math.floor(earlyGroundY - 80), charState.robotX || 640, animFrame);
2736
+ }
2737
+ // ── Ground-level atmospheric haze for depth ──
2738
+ if (world.ground !== 'space') {
2739
+ const hazeY = earlyGroundY - 20;
2740
+ const hazeGrad = ctx.createLinearGradient(0, hazeY - 40, 0, hazeY + 60);
2741
+ hazeGrad.addColorStop(0, 'rgba(13,17,23,0)');
2742
+ hazeGrad.addColorStop(0.5, 'rgba(30,40,50,0.15)');
2743
+ hazeGrad.addColorStop(1, 'rgba(13,17,23,0)');
2744
+ ctx.fillStyle = hazeGrad;
2745
+ ctx.fillRect(0, hazeY - 40, WIDTH, 100);
2746
+ }
2747
+ // ── HACK 1: Stars in the sky ──
2748
+ for (let i = 0; i < 40; i++) {
2749
+ const sx = (i * 97 + 31) % WIDTH;
2750
+ const sy = 40 + (i * 137 + 53) % Math.max(100, earlyGroundY - 100);
2751
+ const brightness = 0.3 + Math.sin(animFrame * (0.08 + (i % 7) * 0.03) + i * 1.7) * 0.4 + 0.3;
2752
+ const size = i < 5 ? 2 : 1;
2753
+ ctx.fillStyle = `rgba(255,255,${200 + (i % 55)},${brightness})`;
2754
+ ctx.fillRect(sx, sy, size, size);
2755
+ }
2381
2756
  // Weather particles over the full frame
2382
2757
  for (const p of world.particles) {
2383
2758
  if (world.weather === 'rain') {
@@ -2402,6 +2777,113 @@ function renderFrame() {
2402
2777
  }
2403
2778
  }
2404
2779
  // ════════════════════════════════════════════════════════════════
2780
+ // LAYER 1.5: AMBIENT SCENERY — trees, rocks, grass, flowers, clouds
2781
+ // ════════════════════════════════════════════════════════════════
2782
+ {
2783
+ const biome = world.ground;
2784
+ const hasVegetation = biome === 'grass' || biome === 'city';
2785
+ const hasAnything = biome !== 'space' && biome !== 'ocean';
2786
+ const camX = charState.robotX || 640;
2787
+ // Deterministic hash: returns 0..1 for a given seed
2788
+ const hash = (seed) => ((Math.sin(seed * 9301 + 4917) * 49297) % 1 + 1) % 1;
2789
+ // ── Clouds (all biomes except space) ──
2790
+ if (biome !== 'space') {
2791
+ for (let i = 0; i < 4; i++) {
2792
+ const baseX = hash(i * 73 + 11) * 1600 - 160;
2793
+ const cy = 40 + hash(i * 47 + 3) * 80;
2794
+ const speed = 0.3 + hash(i * 19 + 7) * 0.7;
2795
+ const cx = (baseX + animFrame * speed) % (WIDTH + 200) - 100;
2796
+ const cw = 60 + hash(i * 31 + 5) * 60;
2797
+ ctx.fillStyle = biome === 'lava' ? 'rgba(80,40,30,0.25)' : 'rgba(255,255,255,0.15)';
2798
+ ctx.beginPath();
2799
+ ctx.ellipse(cx, cy, cw / 2, 12 + hash(i * 53) * 8, 0, 0, Math.PI * 2);
2800
+ ctx.fill();
2801
+ ctx.beginPath();
2802
+ ctx.ellipse(cx - cw * 0.2, cy + 4, cw * 0.3, 10, 0, 0, Math.PI * 2);
2803
+ ctx.fill();
2804
+ }
2805
+ }
2806
+ // ── Trees (grass / city only) ──
2807
+ if (hasVegetation) {
2808
+ for (let i = 0; i < 10; i++) {
2809
+ const wx = hash(i * 131 + 41) * 2400 - 400; // world-space x
2810
+ const tx = ((wx - camX + 3000) % 2400) - 560; // screen x with parallax wrap
2811
+ const depth = 0.5 + hash(i * 67 + 9) * 0.5; // 0.5=far, 1=near
2812
+ const treeH = Math.floor(20 + 25 * depth);
2813
+ const ty = earlyGroundY - treeH * 0.3 - (1 - depth) * 50; // further back = higher
2814
+ const variant = Math.floor(hash(i * 89 + 23) * 3);
2815
+ const trunkW = Math.max(2, Math.floor(3 * depth));
2816
+ const trunkH = Math.floor(treeH * 0.45);
2817
+ // Trunk
2818
+ ctx.fillStyle = variant === 2 ? '#d4cfc4' : '#5a3a1a';
2819
+ ctx.fillRect(tx - trunkW / 2, ty + treeH - trunkH, trunkW, trunkH);
2820
+ // Canopy
2821
+ if (variant === 0) { // pine — triangle
2822
+ ctx.fillStyle = '#2d5a27';
2823
+ ctx.beginPath();
2824
+ ctx.moveTo(tx, ty);
2825
+ ctx.lineTo(tx - treeH * 0.3, ty + treeH * 0.6);
2826
+ ctx.lineTo(tx + treeH * 0.3, ty + treeH * 0.6);
2827
+ ctx.fill();
2828
+ }
2829
+ else if (variant === 1) { // oak — round
2830
+ ctx.fillStyle = '#3a7a34';
2831
+ ctx.beginPath();
2832
+ ctx.arc(tx, ty + treeH * 0.3, treeH * 0.32, 0, Math.PI * 2);
2833
+ ctx.fill();
2834
+ }
2835
+ else { // birch — small leaves on white trunk
2836
+ ctx.fillStyle = '#5a9a44';
2837
+ ctx.beginPath();
2838
+ ctx.ellipse(tx, ty + treeH * 0.25, treeH * 0.22, treeH * 0.3, 0, 0, Math.PI * 2);
2839
+ ctx.fill();
2840
+ }
2841
+ }
2842
+ }
2843
+ // ── Rocks (any non-water/space biome) ──
2844
+ if (hasAnything) {
2845
+ for (let i = 0; i < 7; i++) {
2846
+ const rx = ((hash(i * 199 + 77) * 2000 - camX + 3000) % 2000) - 360;
2847
+ const big = hash(i * 61 + 13) > 0.5;
2848
+ const rw = big ? 8 : 4;
2849
+ const rh = big ? 5 : 3;
2850
+ ctx.fillStyle = '#4a5568';
2851
+ ctx.fillRect(rx, earlyGroundY - rh, rw, rh);
2852
+ ctx.fillStyle = '#718096'; // highlight top edge
2853
+ ctx.fillRect(rx, earlyGroundY - rh, rw, 1);
2854
+ }
2855
+ }
2856
+ // ── Grass tufts (grass / city) ──
2857
+ if (hasVegetation) {
2858
+ for (let i = 0; i < 18; i++) {
2859
+ const gx = ((hash(i * 157 + 33) * 2200 - camX + 3000) % 2200) - 460;
2860
+ const sway = Math.sin(animFrame * 0.06 + i * 2.1) * 1.2;
2861
+ ctx.strokeStyle = biome === 'city' ? '#4a7a3a' : '#3a8a2a';
2862
+ ctx.lineWidth = 1;
2863
+ ctx.beginPath();
2864
+ ctx.moveTo(gx, earlyGroundY);
2865
+ ctx.lineTo(gx + sway, earlyGroundY - 3 - hash(i * 43) * 2);
2866
+ ctx.stroke();
2867
+ ctx.beginPath();
2868
+ ctx.moveTo(gx + 2, earlyGroundY);
2869
+ ctx.lineTo(gx + 2 - sway * 0.7, earlyGroundY - 2 - hash(i * 71) * 2);
2870
+ ctx.stroke();
2871
+ }
2872
+ }
2873
+ // ── Flowers (grass only — not lava/space/ocean/city) ──
2874
+ if (biome === 'grass') {
2875
+ const flowerColors = ['#e53e3e', '#ecc94b', '#ed64a6', '#ffffff', '#f6ad55', '#9f7aea'];
2876
+ for (let i = 0; i < 8; i++) {
2877
+ const fx = ((hash(i * 211 + 59) * 2000 - camX + 3000) % 2000) - 360;
2878
+ const color = flowerColors[Math.floor(hash(i * 97 + 17) * flowerColors.length)];
2879
+ ctx.fillStyle = color;
2880
+ ctx.fillRect(fx, earlyGroundY - 3, 2, 2);
2881
+ ctx.fillStyle = '#3a8a2a'; // tiny stem
2882
+ ctx.fillRect(fx, earlyGroundY - 1, 1, 1);
2883
+ }
2884
+ }
2885
+ }
2886
+ // ════════════════════════════════════════════════════════════════
2405
2887
  // LAYER 2: LIGHTING + FOG over the world
2406
2888
  // ════════════════════════════════════════════════════════════════
2407
2889
  {
@@ -2474,13 +2956,25 @@ function renderFrame() {
2474
2956
  ctx.save();
2475
2957
  ctx.globalAlpha = 0.3;
2476
2958
  ctx.globalCompositeOperation = 'lighter';
2477
- drawRobot(ctx, robotScreenX - offset, robotScreenY, robotScale, charState.mood, animFrame, [255, 50, 50], weatherType, isWalking, charState.walkPhase);
2478
- drawRobot(ctx, robotScreenX + offset, robotScreenY, robotScale, charState.mood, animFrame, [50, 50, 255], weatherType, isWalking, charState.walkPhase);
2959
+ if (characterType === 'gorilla') {
2960
+ drawGorilla(ctx, robotScreenX - offset, robotScreenY, robotScale, charState.mood, animFrame, [255, 50, 50]);
2961
+ drawGorilla(ctx, robotScreenX + offset, robotScreenY, robotScale, charState.mood, animFrame, [50, 50, 255]);
2962
+ }
2963
+ else {
2964
+ drawRobot(ctx, robotScreenX - offset, robotScreenY, robotScale, charState.mood, animFrame, [255, 50, 50], weatherType, isWalking, charState.walkPhase);
2965
+ drawRobot(ctx, robotScreenX + offset, robotScreenY, robotScale, charState.mood, animFrame, [50, 50, 255], weatherType, isWalking, charState.walkPhase);
2966
+ }
2479
2967
  ctx.restore();
2480
2968
  }
2481
2969
  renderDamageFlash(ctx, robotScreenX, robotScreenY, robotScale);
2482
- drawRobot(ctx, robotScreenX, robotScreenY, robotScale, charState.mood, animFrame, undefined, weatherType, isWalking, charState.walkPhase);
2483
- drawMoodParticles(ctx, robotScreenX, robotScreenY, robotScale, charState.mood, animFrame);
2970
+ if (characterType === 'gorilla') {
2971
+ drawGorilla(ctx, robotScreenX, robotScreenY, robotScale, charState.mood, animFrame, undefined);
2972
+ drawGorillaParticles(ctx, robotScreenX, robotScreenY, robotScale, charState.mood, animFrame);
2973
+ }
2974
+ else {
2975
+ drawRobot(ctx, robotScreenX, robotScreenY, robotScale, charState.mood, animFrame, undefined, weatherType, isWalking, charState.walkPhase);
2976
+ drawMoodParticles(ctx, robotScreenX, robotScreenY, robotScale, charState.mood, animFrame);
2977
+ }
2484
2978
  // Subsurface scattering
2485
2979
  {
2486
2980
  const sssPanels = buildSubsurfacePanels(robotScreenX, robotScreenY, robotScale, moodColorHex);
@@ -2580,22 +3074,40 @@ function renderFrame() {
2580
3074
  ctx.moveTo(0, 40);
2581
3075
  ctx.lineTo(WIDTH, 40);
2582
3076
  ctx.stroke();
2583
- // Left: "K:BOT LIVE" in 24px bold accent
3077
+ // Left: "K:BOT" in accent + pulsing red LIVE dot
2584
3078
  ctx.font = 'bold 24px "Courier New", monospace';
2585
3079
  ctx.fillStyle = COLORS.accent;
2586
- ctx.fillText('K:BOT LIVE', 12, 28);
2587
- // Center: current segment name
3080
+ ctx.fillText('K:BOT', 12, 28);
3081
+ // Pulsing red LIVE indicator
3082
+ const livePulse = 0.6 + 0.4 * Math.sin(animFrame * 0.15);
3083
+ ctx.beginPath();
3084
+ ctx.arc(130, 22, 5, 0, Math.PI * 2);
3085
+ ctx.fillStyle = `rgba(248,81,73,${livePulse})`;
3086
+ ctx.fill();
3087
+ ctx.fillStyle = COLORS.red;
3088
+ ctx.font = 'bold 16px "Courier New", monospace';
3089
+ ctx.fillText('LIVE', 140, 28);
3090
+ // Center: segment name + weather/time info
2588
3091
  const segLabel = SEGMENT_LABELS[agenda.currentSegment];
3092
+ const weatherLabel = weatherSystem ? `${weatherSystem.getWeather().type.replace('_', ' ')} | ${weatherSystem.getTimeOfDay()}` : '';
3093
+ const centerText = weatherLabel ? `${segLabel} ~ ${weatherLabel}` : segLabel;
2589
3094
  ctx.fillStyle = COLORS.textDim;
2590
3095
  ctx.font = '14px "Courier New", monospace';
2591
- const segW = ctx.measureText(segLabel).width;
2592
- ctx.fillText(segLabel, (WIDTH - segW) / 2, 26);
2593
- // Right: timer
3096
+ const segW = ctx.measureText(centerText).width;
3097
+ ctx.fillText(centerText, (WIDTH - segW) / 2, 26);
3098
+ // Right: timer + viewer count
2594
3099
  const elapsed = Math.floor((Date.now() - charState.startTime) / 1000);
2595
3100
  const timeStr = `${String(Math.floor(elapsed / 3600)).padStart(2, '0')}:${String(Math.floor((elapsed % 3600) / 60)).padStart(2, '0')}:${String(elapsed % 60).padStart(2, '0')}`;
2596
3101
  ctx.fillStyle = COLORS.textDim;
2597
3102
  ctx.font = '16px "Courier New", monospace';
2598
3103
  ctx.fillText(timeStr, WIDTH - 160, 26);
3104
+ // Viewer count badge
3105
+ const viewerCount = Object.keys(memory.users).length;
3106
+ if (viewerCount > 0) {
3107
+ ctx.fillStyle = hexToRgba(COLORS.green, 0.8);
3108
+ ctx.font = 'bold 12px "Courier New", monospace';
3109
+ ctx.fillText(`${viewerCount} viewers`, WIDTH - 260, 26);
3110
+ }
2599
3111
  // ── Audio atmosphere description (top-center, italic, fades) ──
2600
3112
  if (activeAudioDescription) {
2601
3113
  ctx.save();
@@ -2639,9 +3151,11 @@ function renderFrame() {
2639
3151
  if (recent.length > 0) {
2640
3152
  ctx.save();
2641
3153
  ctx.globalAlpha = chatAlpha;
2642
- // Semi-transparent background
2643
- ctx.fillStyle = 'rgba(13,17,23,0.6)';
3154
+ // Semi-transparent background with accent border
3155
+ ctx.fillStyle = 'rgba(13,17,23,0.65)';
2644
3156
  ctx.fillRect(chatOverlayX, chatOverlayY, chatOverlayW, chatOverlayH);
3157
+ ctx.fillStyle = hexToRgba(COLORS.accent, 0.4);
3158
+ ctx.fillRect(chatOverlayX, chatOverlayY, 3, chatOverlayH);
2645
3159
  // Messages
2646
3160
  for (let i = 0; i < recent.length; i++) {
2647
3161
  const msg = recent[i];
@@ -2686,7 +3200,7 @@ function renderFrame() {
2686
3200
  // Measure text to get bubble width
2687
3201
  const speechW = Math.min(maxBubbleW, ctx.measureText(charState.speech).width + 40);
2688
3202
  const bubbleX = Math.floor((WIDTH - speechW) / 2);
2689
- const bubbleY = HEIGHT - 80;
3203
+ const bubbleY = HEIGHT - 130;
2690
3204
  // Word-wrap to calculate height
2691
3205
  const words = charState.speech.split(' ');
2692
3206
  let testLine = '';
@@ -2802,10 +3316,12 @@ function renderFrame() {
2802
3316
  ctx.fillText(recentContent[i].slice(0, 65), collabX + 6, collabY + 42 + i * 13);
2803
3317
  }
2804
3318
  }
2805
- // ── Website URL (bottom-right, subtle) ──
2806
- ctx.fillStyle = hexToRgba(COLORS.accent, 0.6);
2807
- ctx.font = 'bold 13px "Courier New", monospace';
2808
- ctx.fillText('kernel.chat', WIDTH - 130, HEIGHT - 10);
3319
+ // ── Website URL (bottom-right, branded) ──
3320
+ ctx.fillStyle = hexToRgba(COLORS.accent, 0.35);
3321
+ ctx.font = 'bold 16px "Courier New", monospace';
3322
+ ctx.fillText('kernel.chat', WIDTH - 140, HEIGHT - 14);
3323
+ ctx.fillStyle = hexToRgba(COLORS.accent, 0.8);
3324
+ ctx.fillRect(WIDTH - 142, HEIGHT - 10, 130, 2);
2809
3325
  // ════════════════════════════════════════════════════════════════
2810
3326
  // LAYER 6: ON-DEMAND PANELS (shown for 5 seconds when triggered)
2811
3327
  // ════════════════════════════════════════════════════════════════
@@ -2898,6 +3414,15 @@ function renderFrame() {
2898
3414
  ctx.lineWidth = 2;
2899
3415
  ctx.strokeRect(1, 1, WIDTH - 2, HEIGHT - 2);
2900
3416
  // ════════════════════════════════════════════════════════════════
3417
+ // LAYER 7.5: WEATHER PARTICLES + OVERLAY + COMMANDS
3418
+ // ════════════════════════════════════════════════════════════════
3419
+ if (weatherSystem)
3420
+ weatherSystem.render(ctx, WIDTH, HEIGHT);
3421
+ if (streamCommands)
3422
+ streamCommands.render(ctx, WIDTH, HEIGHT);
3423
+ if (overlay)
3424
+ overlay.render(ctx, WIDTH, HEIGHT);
3425
+ // ════════════════════════════════════════════════════════════════
2901
3426
  // LAYER 8: POST-PROCESSING
2902
3427
  // ════════════════════════════════════════════════════════════════
2903
3428
  {
@@ -2994,6 +3519,12 @@ function startChatPoll() {
2994
3519
  // Track viewer in social engine
2995
3520
  if (socialEngine)
2996
3521
  trackViewer(socialEngine, msg.username, msg.platform, msg.text);
3522
+ // VOD — track chat for highlight detection
3523
+ if (vodSystem && vodSystem.isRecording())
3524
+ vodSystem.onChatMessage(msg.username);
3525
+ // Audio SFX — blip on chat
3526
+ if (audioEngine)
3527
+ triggerSFX(audioEngine, 'chat');
2997
3528
  // Phase 1: !sleep command — trigger dreaming mode
2998
3529
  if (msg.text.toLowerCase().trim() === '!sleep') {
2999
3530
  charState.mood = 'dreaming';
@@ -3071,6 +3602,35 @@ function startChatPoll() {
3071
3602
  }
3072
3603
  }
3073
3604
  }
3605
+ // Stream commands (!duel, !slots, !inventory, !weather vote, etc.)
3606
+ let cmdResult = null;
3607
+ if (streamCommands && !brainResult && !intelResult && !tileResult && !worldResult) {
3608
+ const cr = streamCommands.handleMessage(msg.username, msg.text, msg.platform);
3609
+ if (cr) {
3610
+ cmdResult = cr.response;
3611
+ // Trigger overlay alerts for special events
3612
+ if (overlay && cr.response.includes('JACKPOT')) {
3613
+ overlay.queueAlert({ type: 'achievement', username: msg.username, title: 'JACKPOT!', message: cr.response });
3614
+ }
3615
+ if (overlay && cr.response.includes('BOSS')) {
3616
+ overlay.queueAlert({ type: 'raid', username: msg.username, title: 'BOSS FIGHT!', message: 'A boss has appeared!' });
3617
+ }
3618
+ // Audio SFX for command results
3619
+ if (audioEngine) {
3620
+ if (cr.response.includes('win') || cr.response.includes('won'))
3621
+ triggerSFX(audioEngine, 'achievement');
3622
+ if (cr.response.includes('BOSS'))
3623
+ triggerSFX(audioEngine, 'boss');
3624
+ }
3625
+ }
3626
+ }
3627
+ // Weather system commands (!weather, !time)
3628
+ let weatherResult = null;
3629
+ if (weatherSystem && !brainResult && !intelResult && !tileResult && !worldResult && !cmdResult) {
3630
+ const weatherCmd = weatherSystem.handleCommand(msg.text, msg.text.slice(1));
3631
+ if (weatherCmd && weatherCmd !== msg.text)
3632
+ weatherResult = weatherCmd;
3633
+ }
3074
3634
  // React
3075
3635
  charState.mood = 'talking';
3076
3636
  const responsePromise = brainResult
@@ -3081,7 +3641,13 @@ function startChatPoll() {
3081
3641
  ? Promise.resolve(tileResult)
3082
3642
  : worldResult
3083
3643
  ? Promise.resolve(worldResult)
3084
- : generateResponse(msg.username, msg.text, msg.platform);
3644
+ : cmdResult
3645
+ ? Promise.resolve(cmdResult)
3646
+ : weatherResult
3647
+ ? Promise.resolve(weatherResult)
3648
+ : chatAI
3649
+ ? chatAI.processMessage(msg.username, msg.text, msg.platform).then(r => r || generateResponse(msg.username, msg.text, msg.platform))
3650
+ : generateResponse(msg.username, msg.text, msg.platform);
3085
3651
  responsePromise.then(response => {
3086
3652
  queueSpeech(`@${msg.username}: ${response}`, 'talking', 80, 48, 'chat-response');
3087
3653
  memory.totalResponses++;
@@ -3451,23 +4017,26 @@ function startStream(platforms) {
3451
4017
  const tee = platforms.map(p => `[f=flv]${p.endpoint}/${p.key}`).join('|');
3452
4018
  outputArgs = ['-f', 'tee', tee];
3453
4019
  }
4020
+ const usePCMAudio = audioEngine?.pcmEnabled ?? false;
3454
4021
  const ffmpegArgs = [
3455
4022
  '-f', 'rawvideo', '-pix_fmt', 'rgb24',
3456
4023
  '-s', `${WIDTH}x${HEIGHT}`, '-r', String(FPS),
3457
4024
  '-i', 'pipe:0',
3458
- '-f', 'lavfi', '-i', 'anullsrc=r=44100:cl=stereo',
3459
- '-c:v', 'libx264', '-preset', 'veryfast',
3460
- '-b:v', '2000k', '-maxrate', '2000k', '-bufsize', '4000k',
3461
- '-g', String(FPS * 2), '-keyint_min', String(FPS * 2),
3462
- '-pix_fmt', 'yuv420p',
3463
- '-c:a', 'aac', '-b:a', '128k', '-ar', '44100',
3464
- '-shortest',
3465
4025
  ];
4026
+ // If PCM audio, use pipe:3. Otherwise use anullsrc.
4027
+ if (usePCMAudio) {
4028
+ ffmpegArgs.push('-f', 'f32le', '-ar', '44100', '-ac', '1', '-i', 'pipe:3');
4029
+ }
4030
+ else {
4031
+ ffmpegArgs.push('-f', 'lavfi', '-i', 'anullsrc=r=44100:cl=stereo');
4032
+ }
4033
+ ffmpegArgs.push('-c:v', 'libx264', '-preset', 'veryfast', '-b:v', '2000k', '-maxrate', '2000k', '-bufsize', '4000k', '-g', String(FPS * 2), '-keyint_min', String(FPS * 2), '-pix_fmt', 'yuv420p', '-c:a', 'aac', '-b:a', '128k', '-ar', '44100', '-shortest');
3466
4034
  if (platforms.length > 1) {
3467
4035
  ffmpegArgs.push('-map', '0:v', '-map', '1:a');
3468
4036
  }
3469
4037
  ffmpegArgs.push(...outputArgs);
3470
- const proc = spawn('ffmpeg', ffmpegArgs, { stdio: ['pipe', 'pipe', 'pipe'] });
4038
+ const stdioConfig = usePCMAudio ? ['pipe', 'pipe', 'pipe', 'pipe'] : ['pipe', 'pipe', 'pipe'];
4039
+ const proc = spawn('ffmpeg', ffmpegArgs, { stdio: stdioConfig });
3471
4040
  frameTimer = setInterval(() => {
3472
4041
  if (!proc.stdin || proc.stdin.destroyed)
3473
4042
  return;
@@ -3477,6 +4046,16 @@ function startStream(platforms) {
3477
4046
  charState.frameCount++;
3478
4047
  }
3479
4048
  catch { }
4049
+ // Write real PCM audio samples to pipe:3 when available
4050
+ if (usePCMAudio && audioEngine && audioEngine.pcmEnabled && proc.stdio[3] && !proc.stdio[3].destroyed) {
4051
+ const samplesPerFrame = Math.floor(44100 / FPS); // ~7350 samples per frame at 6fps
4052
+ const audioBuf = generateAudioBuffer(audioEngine, samplesPerFrame, 44100);
4053
+ const buffer = Buffer.from(audioBuf.buffer, audioBuf.byteOffset, audioBuf.byteLength);
4054
+ try {
4055
+ proc.stdio[3].write(buffer);
4056
+ }
4057
+ catch { }
4058
+ }
3480
4059
  }, 1000 / FPS);
3481
4060
  return proc;
3482
4061
  }
@@ -3548,8 +4127,19 @@ export function registerStreamRendererTools() {
3548
4127
  evolutionEngine = loadEvolutionState() || initEvolutionEngine();
3549
4128
  narrativeEngine = loadNarrative() || createNarrativeEngine();
3550
4129
  audioEngine = createAudioEngine();
4130
+ audioEngine.pcmEnabled = true;
4131
+ audioEngine.musicEnabled = true;
3551
4132
  socialEngine = loadSocialEngine();
3552
4133
  activeAudioDescription = null;
4134
+ // Init new systems (v2)
4135
+ overlay = getOverlay();
4136
+ weatherSystem = getWeatherSystem();
4137
+ chatAI = new StreamChatAI();
4138
+ chatAI.loadMemory();
4139
+ vodSystem = new StreamVOD();
4140
+ vodSystem.loadState();
4141
+ streamCommands = getStreamCommands();
4142
+ streamCommands.loadState();
3553
4143
  agenda = {
3554
4144
  currentIndex: 0,
3555
4145
  currentSegment: 'welcome',
@@ -3602,6 +4192,15 @@ export function registerStreamRendererTools() {
3602
4192
  saveNarrative(narrativeEngine);
3603
4193
  if (socialEngine)
3604
4194
  saveSocialEngine(socialEngine);
4195
+ // Save new systems
4196
+ if (chatAI)
4197
+ chatAI.saveMemory();
4198
+ if (vodSystem && vodSystem.isRecording())
4199
+ vodSystem.stopRecording();
4200
+ if (vodSystem)
4201
+ vodSystem.saveState();
4202
+ if (streamCommands)
4203
+ streamCommands.saveState();
3605
4204
  const elapsed = Math.floor((Date.now() - charState.startTime) / 60000);
3606
4205
  return `Stream stopped after ${elapsed}m.\nFrames: ${charState.frameCount}\nMessages: ${memory.totalMessages}\nUsers learned: ${Object.keys(memory.users).length}\nFacts: ${memory.sessionFacts.length}\nSegments completed: ${agenda.currentIndex}`;
3607
4206
  },