neoagent 2.3.1-beta.0 → 2.3.1-beta.100

package/server/services/ai/hooks.js

@@ -0,0 +1,127 @@
+/**
+ * hooks.js — Agent loop lifecycle hook system
+ *
+ * Inspired by OpenClaw's plugin hook architecture. Hooks let integrations,
+ * skills, and agent configs reshape context, observe state, or block
+ * specific operations without touching engine.js core.
+ *
+ * ── WIRED in engine.js ─────────────────────────────────────────────────────
+ *
+ *   before_tool_call(ctx: { toolName, toolArgs, runId, userId, agentId, iteration })
+ *     → Blockable. Return { block: true } to skip the tool call (soft skip,
+ *       not counted as a failure). Return { toolArgs } to mutate arguments.
+ *     Context: fires before DB insert and before executeTool().
+ *
+ *   on_loop_end(ctx: { userId, runId, agentId, status, iterations, totalTokens, taskAnalysis, finalContent })
+ *     → Observer. Fires fire-and-forget after every completed run.
+ *       Use for self-improvement, memory consolidation, analytics.
+ *       Errors are swallowed — this hook must not affect run outcome.
+ *
+ * ── NOT YET WIRED (planned) ────────────────────────────────────────────────
+ *
+ *   before_prompt_build — inject extra system messages before model call
+ *   after_tool_call     — observe/transform tool result after execution
+ *   on_loop_iteration   — called at the top of each iteration; can inject steering
+ *
+ *   To wire one, call globalHooks.run(event, ctx) at the relevant point in
+ *   engine.js and handle the returned object. Follow the before_tool_call
+ *   pattern as a template.
+ *
+ * ── Usage ──────────────────────────────────────────────────────────────────
+ *
+ *   const { globalHooks } = require('./hooks');
+ *
+ *   globalHooks.register('before_tool_call', async (ctx) => {
+ *     if (ctx.toolName === 'execute_command' && ctx.userId === 'restricted') {
+ *       return { block: true };
+ *     }
+ *   }, { priority: 10, id: 'command-guard' });
+ *
+ *   globalHooks.register('on_loop_end', async (ctx) => {
+ *     // fire-and-forget: distill learnings, update memory, post analytics
+ *   }, { id: 'self-improve' });
+ */
+
+class AgentHooks {
+  constructor() {
+    /** @type {Map<string, Array<{fn: Function, priority: number, id: string}>>} */
+    this._hooks = new Map();
+  }
+
+  /**
+   * Register a hook handler.
+   *
+   * @param {string}   event     - Hook event name
+   * @param {Function} fn        - async (ctx) => result | void
+   * @param {object}   [opts]
+   * @param {number}   [opts.priority=50] - Lower fires first
+   * @param {string}   [opts.id]          - Unique ID for deregistration/tracing
+   */
+  register(event, fn, { priority = 50, id } = {}) {
+    if (typeof fn !== 'function') throw new TypeError(`Hook handler for "${event}" must be a function`);
+    const hookId = id ?? `hook_${Date.now()}_${Math.random().toString(36).slice(2)}`;
+    if (!this._hooks.has(event)) this._hooks.set(event, []);
+    const handlers = this._hooks.get(event);
+    handlers.push({ fn, priority, id: hookId });
+    handlers.sort((a, b) => a.priority - b.priority);
+    return hookId;
+  }
+
+  /**
+   * Deregister a hook by ID.
+   */
+  deregister(event, id) {
+    if (!this._hooks.has(event)) return false;
+    const handlers = this._hooks.get(event);
+    const idx = handlers.findIndex((h) => h.id === id);
+    if (idx === -1) return false;
+    handlers.splice(idx, 1);
+    return true;
+  }
+
+  /**
+   * Run all handlers for an event, merging their return values.
+   * If any handler returns { block: true }, short-circuits and returns { block: true }.
+   *
+   * @param {string} event
+   * @param {object} ctx   - Context passed to every handler
+   * @returns {Promise<object>} Merged result from all handlers
+   */
+  async run(event, ctx) {
+    const handlers = this._hooks.get(event) ?? [];
+    let merged = {};
+    for (const { fn, id } of handlers) {
+      let result;
+      try {
+        result = await fn(ctx);
+      } catch (err) {
+        console.warn(`[Hooks] Handler "${id}" for "${event}" threw:`, err.message);
+        continue; // don't let a bad hook crash the loop
+      }
+      if (result?.block === true) return { block: true };
+      if (result && typeof result === 'object') {
+        merged = { ...merged, ...result };
+      }
+    }
+    return merged;
+  }
+
+  /** True if any handlers are registered for this event. */
+  has(event) {
+    return (this._hooks.get(event)?.length ?? 0) > 0;
+  }
+
+  /** List registered hook IDs for an event (useful for debugging). */
+  list(event) {
+    return (this._hooks.get(event) ?? []).map((h) => ({ id: h.id, priority: h.priority }));
+  }
+}
+
+/**
+ * Global hook registry shared across all runs.
+ * Plugins and integrations register here at startup.
+ * Per-run scoped hooks can be created with `new AgentHooks()`.
+ */
+const globalHooks = new AgentHooks();
+
+module.exports = { AgentHooks, globalHooks };

package/server/services/ai/integrated_tools/index.js

@@ -0,0 +1,98 @@
+'use strict';
+
+const { generateSlideDeck } = require('./slidev');
+const { generateVideoWithRemotion } = require('./remotion');
+
+function getIntegratedToolDefinitions() {
+  return [
+    {
+      name: 'generate_slide_deck',
+      description: 'Generate a polished presentation using Slidev and export finished artifacts. Prefer this for decks instead of raw file-writing. Best practice: pass a clear title and a complete slides array, and request export_formats ["pdf"] or ["pdf","pptx"] for a shareable final result.',
+      parameters: {
+        type: 'object',
+        properties: {
+          title: { type: 'string', description: 'Deck title.' },
+          subtitle: { type: 'string', description: 'Optional deck subtitle or framing line.' },
+          theme: { type: 'string', description: 'Slidev theme name. Defaults to "default".' },
+          deck_markdown: { type: 'string', description: 'Optional full Slidev markdown deck. Use this when you want exact Slidev syntax control.' },
+          slides: {
+            type: 'array',
+            description: 'Structured slide definitions. Use this for most decks if you do not need custom Slidev markdown.',
+            items: {
+              type: 'object',
+              properties: {
+                title: { type: 'string', description: 'Slide title.' },
+                body: { type: 'string', description: 'Short narrative paragraph or statement.' },
+                bullets: { type: 'array', items: { type: 'string' }, description: 'Bullet list for the slide.' },
+                notes: { type: 'string', description: 'Presenter notes.' },
+                image_url: { type: 'string', description: 'Optional remote image URL to embed.' },
+                image_path: { type: 'string', description: 'Optional absolute local image path to embed.' },
+                layout: { type: 'string', description: 'Optional Slidev layout, for example cover, section, statement, quote, or two-cols.' },
+                className: { type: 'string', description: 'Optional Slidev class value.' },
+              },
+            },
+          },
+          export_formats: {
+            type: 'array',
+            items: { type: 'string', enum: ['pdf', 'pptx', 'png'] },
+            description: 'Finished output formats. Defaults to ["pdf"].',
+          },
+          filename_base: { type: 'string', description: 'Optional output filename base.' },
+        },
+        required: ['title'],
+      },
+    },
+    {
+      name: 'generate_video_with_remotion',
+      description: 'Generate a finished MP4 video using Remotion. Prefer this for explainers, launch videos, reels, and narrated visual summaries. Best practice: pass 3-10 scenes with explicit duration_seconds, concise on-screen text, and optional image_path or image_url assets.',
+      parameters: {
+        type: 'object',
+        properties: {
+          title: { type: 'string', description: 'Video title or opening headline.' },
+          subtitle: { type: 'string', description: 'Optional supporting line.' },
+          style: { type: 'string', description: 'High-level visual style direction.' },
+          aspect_ratio: { type: 'string', enum: ['16:9', '9:16', '1:1', '4:5'], description: 'Video canvas aspect ratio. Defaults to 16:9.' },
+          fps: { type: 'number', description: 'Frames per second. Defaults to 30.' },
+          audio_path: { type: 'string', description: 'Optional absolute local path to a soundtrack or voiceover file.' },
+          scenes: {
+            type: 'array',
+            description: 'Scene list in playback order.',
+            items: {
+              type: 'object',
+              properties: {
+                title: { type: 'string', description: 'Scene headline.' },
+                body: { type: 'string', description: 'Supporting sentence or paragraph.' },
+                bullets: { type: 'array', items: { type: 'string' }, description: 'Optional bullets to show in the scene.' },
+                duration_seconds: { type: 'number', description: 'Scene duration in seconds.' },
+                image_url: { type: 'string', description: 'Optional remote image URL.' },
+                image_path: { type: 'string', description: 'Optional absolute local image path.' },
+                accent_color: { type: 'string', description: 'Optional accent color, for example #7dd3fc.' },
+                background_color: { type: 'string', description: 'Optional scene background color.' },
+                align: { type: 'string', enum: ['left', 'center', 'right'], description: 'Text alignment.' },
+              },
+              required: ['title'],
+            },
+          },
+          filename_base: { type: 'string', description: 'Optional output filename base.' },
+        },
+        required: ['title', 'scenes'],
+      },
+    },
+  ];
+}
+
+async function executeIntegratedTool(toolName, args, context = {}) {
+  switch (String(toolName || '').trim()) {
+    case 'generate_slide_deck':
+      return generateSlideDeck(args, context);
+    case 'generate_video_with_remotion':
+      return generateVideoWithRemotion(args, context);
+    default:
+      return null;
+  }
+}
+
+module.exports = {
+  executeIntegratedTool,
+  getIntegratedToolDefinitions,
+};

package/server/services/ai/integrated_tools/remotion.js

@@ -0,0 +1,373 @@
+'use strict';
+
+const path = require('path');
+const {
+  copyAssetIntoJob,
+  createArtifactDescriptor,
+  createJobDir,
+  ensureDir,
+  normalizeFilenameBase,
+  promoteArtifactDescriptor,
+  resolveRepoBinary,
+  runCheckedCommand,
+  shellEscape,
+  writeJsonFile,
+  writeTextFile,
+} = require('./shared');
+
+const REMOTION_BIN = resolveRepoBinary('remotion');
+const ASPECT_RATIOS = {
+  '16:9': { width: 1920, height: 1080 },
+  '9:16': { width: 1080, height: 1920 },
+  '1:1': { width: 1080, height: 1080 },
+  '4:5': { width: 1080, height: 1350 },
+};
+
+function normalizeScene(scene = {}) {
+  const durationSeconds = Math.max(1.5, Number(scene.duration_seconds ?? scene.durationSeconds) || 3);
+  return {
+    title: String(scene.title || '').trim(),
+    body: String(scene.body || '').trim(),
+    bullets: Array.isArray(scene.bullets)
+      ? scene.bullets.map((item) => String(item || '').trim()).filter(Boolean).slice(0, 5)
+      : [],
+    durationSeconds,
+    accentColor: String(scene.accent_color || scene.accentColor || '#7dd3fc').trim() || '#7dd3fc',
+    backgroundColor: String(scene.background_color || scene.backgroundColor || '#08111f').trim() || '#08111f',
+    align: ['left', 'center', 'right'].includes(String(scene.align || '').trim())
+      ? String(scene.align).trim()
+      : 'left',
+    imagePath: String(scene.image_path || '').trim(),
+    imageUrl: String(scene.image_url || '').trim(),
+  };
+}
+
+function normalizeProps(args, jobDir, workspaceManager, userId) {
+  const assetsDir = ensureDir(path.join(jobDir, 'public', 'assets'));
+  const scenes = (Array.isArray(args.scenes) ? args.scenes : [])
+    .map(normalizeScene)
+    .filter((scene) => scene.title || scene.body || scene.bullets.length > 0);
+  if (scenes.length === 0) {
+    throw new Error('generate_video_with_remotion requires a non-empty scenes array.');
+  }
+
+  const preparedScenes = scenes.map((scene, index) => {
+    let imageSrc = scene.imageUrl || '';
+    if (!imageSrc && scene.imagePath) {
+      const asset = copyAssetIntoJob(scene.imagePath, assetsDir, `scene-${index + 1}`, workspaceManager, userId);
+      imageSrc = `/assets/${asset.relativePath}`;
+    }
+    return {
+      ...scene,
+      imageSrc,
+    };
+  });
+
+  let audioSrc = '';
+  if (args.audio_path) {
+    const asset = copyAssetIntoJob(args.audio_path, assetsDir, 'soundtrack', workspaceManager, userId);
+    audioSrc = `/assets/${asset.relativePath}`;
+  }
+
+  return {
+    title: String(args.title || 'NeoAgent video').trim(),
+    subtitle: String(args.subtitle || '').trim(),
+    style: String(args.style || 'editorial cinematic').trim(),
+    fps: Math.max(12, Math.min(60, Number(args.fps) || 30)),
+    aspectRatio: ASPECT_RATIOS[String(args.aspect_ratio || '').trim()] ? String(args.aspect_ratio).trim() : '16:9',
+    scenes: preparedScenes,
+    audioSrc,
+  };
+}
+
+function buildRootFile() {
+  return `const React = require('react');
+const {registerRoot, Composition} = require('remotion');
+const props = require('./props.json');
+const {NeoAgentVideo} = require('./VideoComposition');
+
+const sizes = {
+  '16:9': {width: 1920, height: 1080},
+  '9:16': {width: 1080, height: 1920},
+  '1:1': {width: 1080, height: 1080},
+  '4:5': {width: 1080, height: 1350},
+};
+
+const fps = Number(props.fps) || 30;
+const sceneFrames = (props.scenes || []).map((scene) => Math.max(1, Math.round((Number(scene.durationSeconds) || 3) * fps)));
+const durationInFrames = sceneFrames.reduce((sum, next) => sum + next, 0);
+const size = sizes[props.aspectRatio] || sizes['16:9'];
+
+const RemotionRoot = () => (
+  React.createElement(
+    React.Fragment,
+    null,
+    React.createElement(Composition, {
+      id: 'NeoAgentVideo',
+      component: NeoAgentVideo,
+      durationInFrames,
+      fps,
+      width: size.width,
+      height: size.height,
+      defaultProps: props,
+    }),
+  )
+);
+
+registerRoot(RemotionRoot);
+`;
+}
+
+function buildCompositionFile() {
+  return `const React = require('react');
+const {AbsoluteFill, Audio, Img, Sequence, interpolate, spring, useCurrentFrame, useVideoConfig} = require('remotion');
+
+const FONT_STACK = 'Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif';
+
+const shellStyle = {
+  position: 'relative',
+  overflow: 'hidden',
+  fontFamily: FONT_STACK,
+  color: '#f8fafc',
+  padding: 72,
+};
+
+const backdropStyle = {
+  position: 'absolute',
+  inset: 0,
+  background: 'radial-gradient(circle at top left, rgba(125,211,252,0.28), transparent 34%), radial-gradient(circle at bottom right, rgba(168,85,247,0.18), transparent 40%)',
+};
+
+const cardStyle = {
+  position: 'relative',
+  marginTop: 40,
+  borderRadius: 36,
+  padding: '40px 44px',
+  border: '1px solid rgba(255,255,255,0.14)',
+  background: 'rgba(7, 14, 25, 0.68)',
+  boxShadow: '0 28px 90px rgba(0,0,0,0.28)',
+  backdropFilter: 'blur(18px)',
+};
+
+const SceneView = ({scene}) => {
+  const frame = useCurrentFrame();
+  const {fps, durationInFrames} = useVideoConfig();
+  const entrance = spring({frame, fps, config: {damping: 18, mass: 0.9}});
+  const exitOpacity = interpolate(frame, [Math.max(0, durationInFrames - 18), durationInFrames], [1, 0], {
+    extrapolateLeft: 'clamp',
+    extrapolateRight: 'clamp',
+  });
+  const translateY = interpolate(entrance, [0, 1], [34, 0]);
+  const textAlign = scene.align || 'left';
+
+  return (
+    React.createElement(AbsoluteFill, {
+      style: {
+        ...shellStyle,
+        justifyContent: 'center',
+        background: scene.backgroundColor || '#08111f',
+        opacity: exitOpacity,
+      },
+    },
+      React.createElement('div', {style: backdropStyle}),
+      scene.imageSrc
+        ? React.createElement(Img, {
+            src: scene.imageSrc,
+            style: {
+              position: 'absolute',
+              inset: 0,
+              width: '100%',
+              height: '100%',
+              objectFit: 'cover',
+              opacity: 0.18,
+            },
+          })
+        : null,
+      React.createElement('div', {
+        style: {
+          ...cardStyle,
+          transform: \`translateY(\${translateY}px)\`,
+          maxWidth: scene.imageSrc ? '62%' : '100%',
+        },
+      },
+        React.createElement('div', {
+          style: {
+            width: 110,
+            height: 8,
+            borderRadius: 999,
+            background: scene.accentColor || '#7dd3fc',
+            marginBottom: 24,
+          },
+        }),
+        React.createElement('h1', {
+          style: {
+            margin: 0,
+            fontSize: 76,
+            lineHeight: 1.02,
+            letterSpacing: '-0.06em',
+            textAlign,
+          },
+        }, scene.title || 'Scene'),
+        scene.body
+          ? React.createElement('p', {
+              style: {
+                margin: '20px 0 0 0',
+                maxWidth: 900,
+                fontSize: 32,
+                lineHeight: 1.35,
+                color: 'rgba(226,232,240,0.92)',
+                textAlign,
+              },
+            }, scene.body)
+          : null,
+        scene.bullets && scene.bullets.length > 0
+          ? React.createElement('ul', {
+              style: {
+                margin: '28px 0 0 0',
+                paddingLeft: textAlign === 'center' ? 24 : 34,
+                fontSize: 28,
+                lineHeight: 1.45,
+                color: 'rgba(226,232,240,0.88)',
+              },
+            }, scene.bullets.map((bullet, index) => React.createElement('li', {
+              key: String(index),
+              style: {marginBottom: 12},
+            }, bullet)))
+          : null,
+      ),
+      scene.imageSrc
+        ? React.createElement('div', {
+            style: {
+              position: 'absolute',
+              right: 72,
+              bottom: 72,
+              top: 72,
+              width: '30%',
+              borderRadius: 32,
+              overflow: 'hidden',
+              border: '1px solid rgba(255,255,255,0.16)',
+              boxShadow: '0 22px 70px rgba(0,0,0,0.28)',
+              transform: \`translateY(\${translateY * 0.7}px)\`,
+            },
+          },
+            React.createElement(Img, {
+              src: scene.imageSrc,
+              style: {
+                width: '100%',
+                height: '100%',
+                objectFit: 'cover',
+              },
+            }),
+          )
+        : null,
+    )
+  );
+};
+
+const NeoAgentVideo = (props) => {
+  const fps = Number(props.fps) || 30;
+  const scenes = Array.isArray(props.scenes) ? props.scenes : [];
+  let cursor = 0;
+
+  return (
+    React.createElement(AbsoluteFill, {style: {backgroundColor: '#020617'}},
+      props.audioSrc ? React.createElement(Audio, {src: props.audioSrc}) : null,
+      scenes.map((scene, index) => {
+        const durationInFrames = Math.max(1, Math.round((Number(scene.durationSeconds) || 3) * fps));
+        const startFrom = cursor;
+        cursor += durationInFrames;
+        return React.createElement(Sequence, {
+          key: String(index),
+          from: startFrom,
+          durationInFrames,
+        }, React.createElement(SceneView, {scene}));
+      }),
+    )
+  );
+};
+
+exports.NeoAgentVideo = NeoAgentVideo;
+`;
+}
+
+async function generateVideoWithRemotion(args, context = {}) {
+  if (!context.cliExecutor || typeof context.cliExecutor.execute !== 'function') {
+    throw new Error('CLI executor is unavailable for Remotion rendering.');
+  }
+  const filenameBase = normalizeFilenameBase(args.filename_base || args.title || 'video', 'video');
+  const workspaceManager = context.workspaceManager;
+  const userId = context.userId;
+  if (!workspaceManager || typeof workspaceManager.getToolingRoot !== 'function') {
+    throw new Error('Workspace manager is unavailable for Remotion rendering.');
+  }
+  if (!((typeof userId === 'string' && userId.trim()) || (typeof userId === 'number' && Number.isInteger(userId) && userId > 0))) {
+    throw new Error('Missing or invalid userId for Remotion rendering.');
+  }
+  const jobDir = await createJobDir('remotion', filenameBase, workspaceManager, userId);
+  const entryPath = path.join(jobDir, 'index.js');
+  const compositionPath = path.join(jobDir, 'VideoComposition.js');
+  const propsPath = path.join(jobDir, 'props.json');
+  const outputPath = path.join(jobDir, `${filenameBase}.mp4`);
+  const props = normalizeProps(args, jobDir, workspaceManager, userId);
+  const size = ASPECT_RATIOS[props.aspectRatio] || ASPECT_RATIOS['16:9'];
+  const durationInFrames = props.scenes.reduce((sum, scene) => (
+    sum + Math.max(1, Math.round(scene.durationSeconds * props.fps))
+  ), 0);
+
+  writeTextFile(entryPath, buildRootFile());
+  writeTextFile(compositionPath, buildCompositionFile());
+  writeJsonFile(propsPath, props);
+
+  const command = [
+    shellEscape(REMOTION_BIN),
+    'render',
+    shellEscape(entryPath),
+    shellEscape('NeoAgentVideo'),
+    shellEscape(outputPath),
+    '--props',
+    shellEscape(propsPath),
+    '--codec',
+    shellEscape('h264'),
+    '--fps',
+    shellEscape(String(props.fps)),
+    '--width',
+    shellEscape(String(size.width)),
+    '--height',
+    shellEscape(String(size.height)),
+    '--duration',
+    shellEscape(String(durationInFrames)),
+    '--overwrite',
+  ].join(' ');
+
+  const result = await runCheckedCommand(context.cliExecutor, command, {
+    cwd: jobDir,
+    timeout: 20 * 60 * 1000,
+    errorPrefix: 'Remotion render failed.',
+  });
+
+  const videoDescriptor = createArtifactDescriptor(outputPath, {
+    kind: 'video',
+    label: path.basename(outputPath),
+    mimeType: 'video/mp4',
+  });
+  const promotedVideo = promoteArtifactDescriptor(videoDescriptor, context.artifactStore, context.userId);
+
+  return {
+    success: true,
+    tool: 'generate_video_with_remotion',
+    title: props.title,
+    artifacts: [promotedVideo],
+    message: 'Generated rendered video.',
+    render: {
+      aspectRatio: props.aspectRatio,
+      fps: props.fps,
+      durationInFrames,
+      sceneCount: props.scenes.length,
+      durationMs: result.durationMs,
+    },
+  };
+}
+
+module.exports = {
+  generateVideoWithRemotion,
+};