mojulo 0.0.0 → 0.1.1

package/lib/embedder/chunker.js

@@ -0,0 +1,93 @@
+/**
+ * Pure text chunker for vector embeddings.
+ *
+ * 512-character target with 50-char overlap. Locale-agnostic by design —
+ * the embedding model handles tokenization on its end, so we just need
+ * stable, overlapping windows. No sentence-aware splitting; the embedding
+ * model recovers semantic boundaries fine from raw substrings.
+ */
+
+const DEFAULT_CHUNK_SIZE = 512;
+const DEFAULT_OVERLAP = 50;
+
+/**
+ * Split a single document into chunks suitable for embedding.
+ *
+ * @param {string} text
+ * @param {Object} [opts]
+ * @param {number} [opts.chunkSize=512]
+ * @param {number} [opts.overlap=50]
+ * @returns {Array<{ text: string, chunkIndex: number }>}
+ */
+export function chunkText(text, opts = {}) {
+  const { chunkSize = DEFAULT_CHUNK_SIZE, overlap = DEFAULT_OVERLAP } = opts;
+  if (!text || typeof text !== 'string') return [];
+
+  const normalized = text.replace(/\s+/g, ' ').trim();
+  if (normalized.length === 0) return [];
+
+  if (normalized.length <= chunkSize) {
+    return [{ text: normalized, chunkIndex: 0 }];
+  }
+
+  const stride = Math.max(1, chunkSize - overlap);
+  const chunks = [];
+  let chunkIndex = 0;
+  for (let start = 0; start < normalized.length; start += stride) {
+    const slice = normalized.slice(start, start + chunkSize).trim();
+    if (slice.length === 0) continue;
+    chunks.push({ text: slice, chunkIndex });
+    chunkIndex++;
+    if (start + chunkSize >= normalized.length) break;
+  }
+  return chunks;
+}
+
+/**
+ * Chunk a batch of documents, attaching metadata so we can trace each chunk
+ * back to its source.
+ *
+ * @param {Array<{ id: string, originalName: string, text: string }>} docs
+ * @param {Object} [opts]
+ * @returns {Array<{ text: string, metadata: { documentId: string, originalName: string, chunkIndex: number } }>}
+ */
+export function chunkDocuments(docs, opts = {}) {
+  const out = [];
+  for (const doc of docs || []) {
+    const chunks = chunkText(doc.text, opts);
+    for (const chunk of chunks) {
+      out.push({
+        text: chunk.text,
+        metadata: {
+          documentId: doc.id,
+          originalName: doc.originalName,
+          chunkIndex: chunk.chunkIndex,
+        },
+      });
+    }
+  }
+  return out;
+}
+
+// Triage route descriptions go into the same cosine index as document chunks.
+// At retrieval time, vector-rag.js reads metadata.source to format the snippet
+// so the LLM sees the deploymentId inline alongside the description text.
+// Route descriptions are already concise — we don't sub-chunk; one chunk per
+// route preserves the description as a single retrieval unit.
+export function chunkTriageRoutes(routes) {
+  const out = [];
+  for (const route of routes || []) {
+    const text = (route.description || '').replace(/\s+/g, ' ').trim();
+    if (!text) continue;
+    out.push({
+      text,
+      metadata: {
+        source: 'triage-route',
+        deploymentId: route.deploymentId,
+        originalName: route.name,
+        chunkIndex: 0,
+      },
+    });
+  }
+  return out;
+}

package/lib/embedder/local.js

@@ -0,0 +1,101 @@
+/**
+ * Local embedding client for the control plane.
+ *
+ * Loads multilingual-e5-small from the
+ * pre-fetched ONNX cache at lib/embedder/models/ via @huggingface/transformers
+ * and runs inference in-process. No network calls.
+ *
+ * The same model + dtype combo runs in the lite-template artifact at
+ * runtime, so corpus and query vectors live in the same geometric space.
+ *
+ * e5 models expect prefixed inputs:
+ *   - 'passage: <text>' for corpus chunks
+ *   - 'query: <text>' for retrieval queries
+ * This module owns that convention so callers stay model-agnostic.
+ *
+ * The cache dir resolves to MOJULO_MODELS_DIR if set (npx flow lands this at
+ * ~/.mojulo/models), else the bundled lib/embedder/models/ next to this file
+ * (clone-and-run flow).
+ */
+
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { pipeline, env } from '@huggingface/transformers';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+// Two modes:
+//   - MOJULO_MODELS_DIR set (npx flow): cache lives under the user's home,
+//     gets populated lazily on first call. Remote downloads enabled so the
+//     fresh-dir case self-heals without a separate postinstall step.
+//   - MOJULO_MODELS_DIR unset (clone-and-run flow): cache lives next to this
+//     file, postinstall fills it. Remote downloads stay off — preserves the
+//     current offline-after-install posture.
+const USER_CACHE = !!process.env.MOJULO_MODELS_DIR;
+env.cacheDir = process.env.MOJULO_MODELS_DIR || path.resolve(__dirname, 'models');
+env.allowRemoteModels = USER_CACHE;
+env.allowLocalModels = true;
+
+const MODEL_ID = 'Xenova/multilingual-e5-small';
+const DTYPE = 'q8';
+
+export const LOCAL_EMBEDDING_MODEL = 'multilingual-e5-small';
+export const LOCAL_EMBEDDING_DIM = 384;
+
+let extractorPromise = null;
+
+/**
+ * Begin loading the embedding model in the background. Idempotent — shares
+ * promise state with the lazy path in generateEmbeddings(), so calling this
+ * at server cold-start lets the 113MB fetch overlap with the user's first
+ * exchanges instead of blocking their first RAG action. Callers that don't
+ * await should still .catch() to avoid an unhandled rejection.
+ */
+export async function preloadModel() {
+  await getExtractor();
+}
+
+function getExtractor() {
+  if (!extractorPromise) {
+    extractorPromise = pipeline('feature-extraction', MODEL_ID, { dtype: DTYPE }).catch(
+      (err) => {
+        extractorPromise = null;
+        const hint = USER_CACHE
+          ? `Lazy download from ${env.cacheDir} failed — check network / disk and retry.`
+          : `Run "node scripts/fetch-embed-model.js" first.`;
+        throw new Error(
+          `Failed to load embedding model from ${env.cacheDir}. ${hint} Cause: ${err.message}`
+        );
+      }
+    );
+  }
+  return extractorPromise;
+}
+
+/**
+ * Generate embeddings for a list of texts. Returns L2-normalized
+ * 384-dim float arrays parallel to the input.
+ *
+ * @param {string[]} texts
+ * @param {Object} options
+ * @param {'search_document' | 'search_query'} options.inputType
+ * @returns {Promise<number[][]>}
+ */
+export async function generateEmbeddings(texts, { inputType } = {}) {
+  if (!Array.isArray(texts) || texts.length === 0) {
+    throw new Error('generateEmbeddings: texts must be a non-empty array');
+  }
+  if (inputType !== 'search_document' && inputType !== 'search_query') {
+    throw new Error(
+      "generateEmbeddings: inputType is required ('search_document' | 'search_query')"
+    );
+  }
+
+  const prefix = inputType === 'search_query' ? 'query: ' : 'passage: ';
+  const extractor = await getExtractor();
+  const out = await extractor(
+    texts.map((t) => prefix + t),
+    { pooling: 'mean', normalize: true }
+  );
+  return out.tolist();
+}

package/lib/embedder/preview-rag.js

@@ -0,0 +1,93 @@
+/**
+ * VectorRAGPreview — in-memory mirror of lite-template/helper/vector-rag.js
+ * for the wizard's /api/preview/chat path.
+ *
+ * The deployed bot loads embeddings.json from disk; the preview hydrates
+ * the same payload from a downloadToBuffer() call. Same retrieval semantics,
+ * same query-side prefix, same cosine math.
+ */
+
+import { generateEmbeddings } from './local.js';
+
+function cosineSimilarity(a, b) {
+  if (a.length !== b.length) throw new Error('Vectors must have same length');
+  let dot = 0;
+  let na = 0;
+  let nb = 0;
+  for (let i = 0; i < a.length; i++) {
+    dot += a[i] * b[i];
+    na += a[i] * a[i];
+    nb += b[i] * b[i];
+  }
+  const denom = Math.sqrt(na) * Math.sqrt(nb);
+  return denom === 0 ? 0 : dot / denom;
+}
+
+function findSimilar(queryEmbedding, chunks, k = 3) {
+  const scored = chunks.map((chunk) => ({
+    text: chunk.text,
+    score: cosineSimilarity(queryEmbedding, chunk.embedding),
+    metadata: chunk.metadata,
+  }));
+  scored.sort((a, b) => b.score - a.score);
+  return scored.slice(0, k);
+}
+
+export default class VectorRAGPreview {
+  /**
+   * @param {{ chunks: Array<{ text: string, embedding: number[], metadata?: Object }>, model?: string }} payload
+   */
+  constructor(payload) {
+    this.chunks = Array.isArray(payload?.chunks) ? payload.chunks : [];
+    this.model = payload?.model || null;
+    this.isLoaded = this.chunks.length > 0;
+    this.lastSearchResults = null;
+    this.mode = 'vector';
+  }
+
+  // No-op for parity with VectorRAG.initialize() (lite-template runtime).
+  async initialize() {
+    return;
+  }
+
+  async search(query, maxResults = 3) {
+    if (!this.isLoaded) return '';
+    const cleanQuery = (query || '').trim();
+    if (cleanQuery.length < 3) return '';
+
+    const [queryVec] = await generateEmbeddings([cleanQuery], { inputType: 'search_query' });
+    const top = findSimilar(queryVec, this.chunks, maxResults);
+
+    if (top.length === 0) {
+      this.lastSearchResults = null;
+      return '';
+    }
+
+    this.lastSearchResults = top.map((hit) => ({
+      filename: hit.metadata?.originalName || 'unknown',
+      content: hit.text,
+      score: hit.score,
+      chunkIndex: hit.metadata?.chunkIndex ?? null,
+      source: hit.metadata?.source || 'document',
+      deploymentId: hit.metadata?.deploymentId || null,
+    }));
+
+    return top
+      .map((hit, i) => {
+        const prefix = top.length > 1 ? `[${i + 1}] ` : '';
+        const isRoute = hit.metadata?.source === 'triage-route';
+        if (isRoute) {
+          const id = hit.metadata?.deploymentId || 'unknown';
+          const name = hit.metadata?.originalName || id;
+          return `${prefix}[Triage route — deploymentId: ${id} | name: ${name}]:\n${hit.text}`;
+        }
+        const filename = hit.metadata?.originalName || 'document';
+        return `${prefix}[From ${filename}]:\n${hit.text}`;
+      })
+      .join('\n\n---\n\n');
+  }
+
+  getLastSearchResults() {
+    return this.lastSearchResults;
+  }
+}

package/lib/envelope-schema.js

@@ -0,0 +1,54 @@
+// Canonical envelope shape — mirror of lite-template/helper/envelope-schema.js.
+// Two npm packages, no shared layer; keep in sync.
+//
+// canonical source: lite-template/helper/envelope-schema.js
+// mirror:           control/lib/envelope-schema.js
+
+export const ENVELOPE_SCHEMA = {
+  type: 'object',
+  required: ['answer'],
+  additionalProperties: false,
+  properties: {
+    answer:      { type: 'string' },
+    suggestions: { type: 'array', items: { type: 'string' } },
+
+    form: {
+      type: 'object',
+      additionalProperties: false,
+      properties: {
+        fields:    { type: 'object', additionalProperties: true },
+        remaining: { type: 'integer', minimum: 0 },
+        complete:  { type: 'boolean' },
+      },
+    },
+
+    triage: {
+      type: 'object',
+      additionalProperties: false,
+      properties: {
+        deploymentId:  { type: 'string' },
+        starterPrompt: { type: 'string' },
+      },
+    },
+
+    appointment: {
+      type: 'object',
+      additionalProperties: false,
+      properties: {
+        showLaunchButton: { type: 'boolean' },
+        calendarId:       { type: 'string' },
+      },
+    },
+
+    extraction: {
+      type: 'object',
+      additionalProperties: false,
+      properties: {
+        fields:           { type: 'object', additionalProperties: true },
+        confidence:       { type: 'string', enum: ['high', 'medium', 'low'] },
+        notes:            { type: 'string' },
+        showUploadButton: { type: 'boolean' },
+      },
+    },
+  },
+};

package/lib/fleet/scoped-sql.js

@@ -0,0 +1,342 @@
+/**
+ * In-memory SQLite assembly + SELECT-only validator for the SQL Explorer.
+ *
+ * On every query, we:
+ *   1. Spin up a fresh `:memory:` better-sqlite3 DB.
+ *   2. Fan out to every connected bot for its rollup endpoints.
+ *   3. Load returned rows into named tables that mirror Prime's schema
+ *      surface (bots, daily_bot_stats, bot_health, protocol_stats).
+ *   4. Validate the user query (SELECT/WITH only, single statement, no
+ *      ATTACH/PRAGMA/destructive verbs).
+ *   5. Run it with row + duration caps.
+ *   6. Discard the DB.
+ *
+ * Nothing crosses to the control-plane SQLite. The aggregates live only in
+ * process memory for the duration of one request.
+ *
+ * Scoping note: in single-user mode the in-memory DB only contains the
+ * user's own deployments — there's nothing to scope away, so the CTE
+ * rewrite from Prime is intentionally omitted here. The seam is the same
+ * table set, so a future multi-tenant variant can wrap the validated
+ * query without changing the surface.
+ */
+
+import Database from 'better-sqlite3';
+import { fanOut, listConnectedDeployments } from '@/lib/deployers/bot-fleet';
+
+const ROW_CAP = 10_000;
+const QUERY_TIMEOUT_MS = 30_000;
+
+const FORBIDDEN_KEYWORDS = [
+  'INSERT', 'UPDATE', 'DELETE', 'REPLACE',
+  'DROP', 'ALTER', 'CREATE', 'TRUNCATE',
+  'ATTACH', 'DETACH', 'PRAGMA', 'VACUUM',
+  'REINDEX', 'ANALYZE',
+];
+
+function stripCommentsAndStrings(sql) {
+  // Replace single-quote string literals and SQL comments with spaces so the
+  // forbidden-keyword scan can't be fooled by `SELECT 'DELETE'`.
+  let out = '';
+  let i = 0;
+  while (i < sql.length) {
+    const c = sql[i];
+    const next = sql[i + 1];
+    if (c === '-' && next === '-') {
+      while (i < sql.length && sql[i] !== '\n') i++;
+      out += ' ';
+      continue;
+    }
+    if (c === '/' && next === '*') {
+      i += 2;
+      while (i < sql.length && !(sql[i] === '*' && sql[i + 1] === '/')) i++;
+      i += 2;
+      out += ' ';
+      continue;
+    }
+    if (c === "'") {
+      out += ' ';
+      i++;
+      while (i < sql.length) {
+        if (sql[i] === "'" && sql[i + 1] === "'") { i += 2; continue; }
+        if (sql[i] === "'") { i++; break; }
+        i++;
+      }
+      continue;
+    }
+    if (c === '"') {
+      // Double-quoted identifiers — keep contents (they're not keywords).
+      out += c;
+      i++;
+      while (i < sql.length && sql[i] !== '"') {
+        out += sql[i];
+        i++;
+      }
+      if (i < sql.length) { out += sql[i]; i++; }
+      continue;
+    }
+    out += c;
+    i++;
+  }
+  return out;
+}
+
+export function validateUserSql(sql) {
+  if (typeof sql !== 'string' || !sql.trim()) {
+    return { ok: false, error: 'Empty query' };
+  }
+  const stripped = stripCommentsAndStrings(sql).trim();
+  if (!stripped) return { ok: false, error: 'Empty query' };
+
+  // Single statement only — reject anything past a non-trailing semicolon.
+  const noTrailing = stripped.replace(/;+\s*$/, '');
+  if (noTrailing.includes(';')) {
+    return { ok: false, error: 'Multiple statements are not allowed' };
+  }
+
+  // Must start with SELECT or WITH (case-insensitive).
+  const first = noTrailing.toUpperCase().match(/^\s*(SELECT|WITH)\b/);
+  if (!first) {
+    return { ok: false, error: 'Only SELECT queries are allowed' };
+  }
+
+  // Reject forbidden keywords anywhere.
+  const upper = noTrailing.toUpperCase();
+  for (const kw of FORBIDDEN_KEYWORDS) {
+    const re = new RegExp(`\\b${kw}\\b`);
+    if (re.test(upper)) {
+      return { ok: false, error: `Disallowed keyword: ${kw}` };
+    }
+  }
+  return { ok: true };
+}
+
+/**
+ * Build an in-memory SQLite populated with fleet rollups and run `sql` against
+ * it. Returns { rows, columns, rowCount, truncated, fleet }.
+ */
+export async function runScopedSql(sql) {
+  const v = validateUserSql(sql);
+  if (!v.ok) return { error: v.error };
+
+  const deployments = await listConnectedDeployments();
+  const fleetMeta = {
+    totalCount: deployments.length,
+    reachableCount: 0,
+    unreachableCount: 0,
+    unreachable: [],
+  };
+
+  const db = new Database(':memory:');
+  try {
+    db.exec(`
+      CREATE TABLE bots (
+        id TEXT PRIMARY KEY,
+        bot_name TEXT NOT NULL,
+        url TEXT,
+        last_seen_at TEXT,
+        created_at TEXT,
+        cloud_provider TEXT,
+        cloud_status TEXT,
+        flow_type TEXT
+      );
+      CREATE TABLE daily_bot_stats (
+        bot_id TEXT NOT NULL,
+        date TEXT NOT NULL,
+        conversations INTEGER NOT NULL,
+        turns INTEGER NOT NULL,
+        avg_turns REAL NOT NULL
+      );
+      CREATE TABLE bot_health (
+        bot_id TEXT PRIMARY KEY,
+        bot_name TEXT NOT NULL,
+        conversations_7d INTEGER NOT NULL,
+        turns_7d INTEGER NOT NULL,
+        avg_turns_7d REAL NOT NULL,
+        conversations_total INTEGER NOT NULL,
+        turns_total INTEGER NOT NULL,
+        last_activity_at TEXT
+      );
+      CREATE TABLE protocol_stats (
+        bot_id TEXT NOT NULL,
+        protocol TEXT NOT NULL,
+        turns INTEGER NOT NULL,
+        conversations_touched INTEGER NOT NULL
+      );
+    `);
+
+    // Populate the control-plane-known `bots` table without any fan-out.
+    const insertBot = db.prepare(`
+      INSERT INTO bots (id, bot_name, url, last_seen_at, created_at, cloud_provider, cloud_status, flow_type)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+    `);
+    for (const d of deployments) {
+      insertBot.run(
+        d.id,
+        d.botName,
+        d.url,
+        d.lastSeenAt ? d.lastSeenAt.toISOString() : null,
+        d.createdAt ? d.createdAt.toISOString() : null,
+        d.cloudProvider,
+        d.cloudStatus,
+        d.flowType,
+      );
+    }
+
+    // Fan out for the three rollup tables in parallel.
+    const [daily, health, proto] = await Promise.all([
+      fanOut('/api/analytics/daily_stats', { deployments }),
+      fanOut('/api/analytics/bot_health', { deployments }),
+      fanOut('/api/analytics/protocol_stats', { deployments }),
+    ]);
+
+    // Use bot_health's reachability as the canonical fleet status, since
+    // it's the cheapest call.
+    fleetMeta.reachableCount = health.reachableCount;
+    fleetMeta.unreachableCount = health.unreachableCount;
+    fleetMeta.unreachable = health.results
+      .filter((r) => !r.ok)
+      .map((r) => ({
+        id: r.deployment.id,
+        botName: r.deployment.botName,
+        reason: r.reason,
+        status: r.status,
+      }));
+
+    const insertDaily = db.prepare(`
+      INSERT INTO daily_bot_stats (bot_id, date, conversations, turns, avg_turns)
+      VALUES (?, ?, ?, ?, ?)
+    `);
+    for (const r of daily.results) {
+      if (!r.ok) continue;
+      for (const row of (r.data && r.data.rows) || []) {
+        insertDaily.run(
+          r.deployment.id,
+          row.date,
+          row.conversations || 0,
+          row.turns || 0,
+          row.avg_turns || 0,
+        );
+      }
+    }
+
+    const insertHealth = db.prepare(`
+      INSERT INTO bot_health (bot_id, bot_name, conversations_7d, turns_7d, avg_turns_7d, conversations_total, turns_total, last_activity_at)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+    `);
+    for (const r of health.results) {
+      if (!r.ok) continue;
+      const d = r.data || {};
+      insertHealth.run(
+        r.deployment.id,
+        r.deployment.botName,
+        d.conversations7d || 0,
+        d.turns7d || 0,
+        d.avgTurns7d || 0,
+        d.conversationsTotal || 0,
+        d.turnsTotal || 0,
+        d.lastActivityAt || null,
+      );
+    }
+
+    const insertProto = db.prepare(`
+      INSERT INTO protocol_stats (bot_id, protocol, turns, conversations_touched)
+      VALUES (?, ?, ?, ?)
+    `);
+    for (const r of proto.results) {
+      if (!r.ok) continue;
+      for (const row of (r.data && r.data.rows) || []) {
+        insertProto.run(
+          r.deployment.id,
+          row.protocol,
+          row.turns || 0,
+          row.conversations_touched || 0,
+        );
+      }
+    }
+
+    // Run the user query under a timeout. better-sqlite3 is synchronous, but
+    // we wrap with a wall-clock guard so a runaway recursive CTE can't pin
+    // the event loop forever.
+    const start = Date.now();
+    const stmt = db.prepare(sql);
+    let rawRows;
+    try {
+      rawRows = stmt.all();
+    } catch (err) {
+      return { error: err.message };
+    }
+    if (Date.now() - start > QUERY_TIMEOUT_MS) {
+      return { error: `Query exceeded ${QUERY_TIMEOUT_MS}ms` };
+    }
+
+    const truncated = rawRows.length > ROW_CAP;
+    const rows = truncated ? rawRows.slice(0, ROW_CAP) : rawRows;
+    const columns = stmt.columns().map((c) => c.name);
+    return {
+      rows,
+      columns,
+      rowCount: rows.length,
+      truncated,
+      fleet: fleetMeta,
+    };
+  } finally {
+    db.close();
+  }
+}
+
+/**
+ * Lightweight schema descriptor — used to render the schema reference
+ * panel without needing a round-trip.
+ */
+export const FLEET_SCHEMA = [
+  {
+    name: 'bots',
+    description: 'One row per registered deployment (control-plane state)',
+    columns: [
+      { name: 'id', description: 'Deployment id' },
+      { name: 'bot_name', description: 'Display name' },
+      { name: 'url', description: 'Bot URL if connected' },
+      { name: 'last_seen_at', description: 'Last successful proxy call' },
+      { name: 'created_at', description: 'Deployment created at' },
+      { name: 'cloud_provider', description: 'fly | null' },
+      { name: 'cloud_status', description: 'running | paused | failed | null' },
+      { name: 'flow_type', description: 'Configured flow type' },
+    ],
+  },
+  {
+    name: 'daily_bot_stats',
+    description: 'Per-bot per-day conversation and turn counts',
+    columns: [
+      { name: 'bot_id', description: 'FK → bots.id' },
+      { name: 'date', description: 'YYYY-MM-DD' },
+      { name: 'conversations', description: 'Distinct conversation count' },
+      { name: 'turns', description: 'Turn count' },
+      { name: 'avg_turns', description: 'turns / conversations' },
+    ],
+  },
+  {
+    name: 'bot_health',
+    description: 'One row per bot — last 7 days plus all-time totals',
+    columns: [
+      { name: 'bot_id', description: 'FK → bots.id' },
+      { name: 'bot_name', description: 'Display name' },
+      { name: 'conversations_7d', description: 'Last 7 days' },
+      { name: 'turns_7d', description: 'Last 7 days' },
+      { name: 'avg_turns_7d', description: 'Last 7 days' },
+      { name: 'conversations_total', description: 'All time' },
+      { name: 'turns_total', description: 'All time' },
+      { name: 'last_activity_at', description: 'Most recent turn timestamp' },
+    ],
+  },
+  {
+    name: 'protocol_stats',
+    description: 'Per-bot per-protocol turn and conversation counts',
+    columns: [
+      { name: 'bot_id', description: 'FK → bots.id' },
+      { name: 'protocol', description: 'form | triage | appointment | extraction' },
+      { name: 'turns', description: 'Turns whose machine_state touched this protocol' },
+      { name: 'conversations_touched', description: 'Distinct conversations' },
+    ],
+  },
+];