mulmocast-preprocessor 0.2.1 → 0.3.0

package/lib/cli/commands/query.js

@@ -1,7 +1,7 @@
 import { createInterface } from "node:readline";
 import { GraphAILogger } from "graphai";
 import { queryScript } from "../../core/ai/command/query/index.js";
-import { createInteractiveSession, sendInteractiveQuery, clearHistory } from "../../core/ai/command/query/interactive.js";
+import { createInteractiveSession, sendInteractiveQuery, sendInteractiveQueryWithFetch, clearHistory, getReferences, fetchReference, parseSuggestedFetch, removeSuggestFetchMarkers, } from "../../core/ai/command/query/interactive.js";
 import { loadScript } from "../utils.js";
 /**
  * Query command handler - outputs answer to stdout
@@ -37,6 +37,12 @@ export const queryCommand = async (scriptPath, question, options) => {
         process.exit(1);
     }
 };
+/**
+ * Format references for display
+ */
+const formatReferences = (references) => {
+    return references.map((ref, i) => `  ${i + 1}. [${ref.type || "web"}] ${ref.title || ref.url}`).join("\n");
+};
 /**
  * Run interactive query mode
  */
@@ -54,13 +60,18 @@ const runInteractiveMode = async (scriptPath, script, options) => {
         GraphAILogger.error("No content available to query.");
         process.exit(1);
     }
+    const references = getReferences(script);
     const rl = createInterface({
         input: process.stdin,
         output: process.stdout,
     });
     GraphAILogger.info(`Interactive query mode for "${session.scriptTitle}" (${session.beatCount} beats)`);
-    GraphAILogger.info("Commands: /clear (clear history), /history (show history), /exit or Ctrl+C (quit)");
+    GraphAILogger.info("Commands: /clear (clear history), /history (show history), /refs (show references), /fetch <url> (fetch URL), /exit (quit)");
+    if (references.length > 0) {
+        GraphAILogger.info(`Available references: ${references.length}`);
+    }
     GraphAILogger.info("");
+    let lastSuggestedUrl = null;
     const prompt = () => {
         rl.question("You: ", async (input) => {
             const trimmedInput = input.trim();
@@ -76,6 +87,7 @@ const runInteractiveMode = async (scriptPath, script, options) => {
             }
             if (trimmedInput === "/clear") {
                 clearHistory(session);
+                lastSuggestedUrl = null;
                 GraphAILogger.info("Conversation history cleared.\n");
                 prompt();
                 return;
@@ -95,10 +107,75 @@ const runInteractiveMode = async (scriptPath, script, options) => {
                 prompt();
                 return;
             }
+            if (trimmedInput === "/refs" || trimmedInput === "/references") {
+                if (references.length === 0) {
+                    GraphAILogger.info("No references available.\n");
+                }
+                else {
+                    GraphAILogger.info("Available references:");
+                    GraphAILogger.info(formatReferences(references));
+                    GraphAILogger.info("");
+                }
+                prompt();
+                return;
+            }
+            // Handle /fetch command
+            if (trimmedInput.startsWith("/fetch")) {
+                const urlArg = trimmedInput.replace(/^\/fetch\s*/, "").trim();
+                const urlToFetch = urlArg || lastSuggestedUrl;
+                if (!urlToFetch) {
+                    GraphAILogger.info("Usage: /fetch <url> or /fetch (to fetch last suggested URL)\n");
+                    prompt();
+                    return;
+                }
+                GraphAILogger.info(`Fetching: ${urlToFetch}...`);
+                try {
+                    const fetchedContent = await fetchReference(urlToFetch, validatedOptions.verbose);
+                    if (fetchedContent.error) {
+                        GraphAILogger.error(`Fetch error: ${fetchedContent.error}\n`);
+                    }
+                    else {
+                        GraphAILogger.info(`Fetched ${fetchedContent.content.length} chars from ${fetchedContent.title || urlToFetch}`);
+                        GraphAILogger.info("Content loaded. Ask a question to use this reference.\n");
+                        // Store for next query
+                        session.fetchedContent = fetchedContent;
+                    }
+                }
+                catch (error) {
+                    if (error instanceof Error) {
+                        GraphAILogger.error(`Fetch error: ${error.message}\n`);
+                    }
+                    else {
+                        GraphAILogger.error("Unknown fetch error\n");
+                    }
+                }
+                prompt();
+                return;
+            }
             // Send query
             try {
-                const answer = await sendInteractiveQuery(filteredScript, trimmedInput, session, validatedOptions);
-                GraphAILogger.info(`\nAssistant: ${answer}\n`);
+                let answer;
+                // If we have fetched content, use it
+                if (session.fetchedContent) {
+                    answer = await sendInteractiveQueryWithFetch(filteredScript, trimmedInput, session.fetchedContent, session, validatedOptions);
+                    // Clear fetched content after use
+                    session.fetchedContent = undefined;
+                }
+                else {
+                    answer = await sendInteractiveQuery(filteredScript, trimmedInput, session, validatedOptions);
+                }
+                // Check for suggested fetch URL
+                const suggestedUrl = parseSuggestedFetch(answer);
+                if (suggestedUrl) {
+                    lastSuggestedUrl = suggestedUrl;
+                    const cleanAnswer = removeSuggestFetchMarkers(answer);
+                    GraphAILogger.info(`\nAssistant: ${cleanAnswer}`);
+                    GraphAILogger.info(`\n(Suggested reference: ${suggestedUrl})`);
+                    GraphAILogger.info("Type /fetch to load this reference for more details.\n");
+                }
+                else {
+                    GraphAILogger.info(`\nAssistant: ${answer}\n`);
+                }
             }
             catch (error) {
                 if (error instanceof Error) {

package/lib/core/ai/command/query/interactive.d.ts

@@ -1,5 +1,6 @@
-import type { ExtendedScript } from "../../../../types/index.js";
+import type { ExtendedScript, Reference } from "../../../../types/index.js";
 import type { QueryOptions, InteractiveQuerySession, ConversationMessage } from "../../../../types/query.js";
+import { type FetchedContent } from "../../utils/fetcher.js";
 /**
  * Create an interactive query session
  */
@@ -20,3 +21,32 @@ export declare const clearHistory: (session: InteractiveQuerySession) => void;
  * Get conversation history
  */
 export declare const getHistory: (session: InteractiveQuerySession) => ConversationMessage[];
+/**
+ * Regex pattern for SUGGEST_FETCH marker (bounded quantifier to prevent ReDoS)
+ */
+export declare const SUGGEST_FETCH_PATTERN: RegExp;
+export declare const SUGGEST_FETCH_PATTERN_GLOBAL: RegExp;
+/**
+ * Parse suggested fetch URL from AI response
+ */
+export declare const parseSuggestedFetch: (response: string) => string | null;
+/**
+ * Remove SUGGEST_FETCH markers from response
+ */
+export declare const removeSuggestFetchMarkers: (response: string) => string;
+/**
+ * Get available references from script
+ */
+export declare const getReferences: (script: ExtendedScript) => Reference[];
+/**
+ * Fetch reference content by URL
+ */
+export declare const fetchReference: (url: string, verbose?: boolean) => Promise<FetchedContent>;
+/**
+ * Find matching reference for a query
+ */
+export declare const findReference: (script: ExtendedScript, query: string) => Reference | null;
+/**
+ * Send a question with fetched reference content
+ */
+export declare const sendInteractiveQueryWithFetch: (filteredScript: ExtendedScript, question: string, fetchedContent: FetchedContent, session: InteractiveQuerySession, options: QueryOptions) => Promise<string>;

package/lib/core/ai/command/query/interactive.js

@@ -1,6 +1,7 @@
 import { queryOptionsSchema } from "../../../../types/query.js";
-import { executeLLM, filterScript } from "../../llm.js";
-import { buildInteractiveUserPrompt, getInteractiveSystemPrompt } from "./prompts.js";
+import { executeLLM, filterScript, getLanguageName } from "../../llm.js";
+import { buildInteractiveUserPrompt, getInteractiveSystemPrompt, DEFAULT_INTERACTIVE_SYSTEM_PROMPT_WITH_FETCH } from "./prompts.js";
+import { fetchUrlContent, findMatchingReference } from "../../utils/fetcher.js";
 /**
  * Create an interactive query session
  */
@@ -42,3 +43,78 @@ export const clearHistory = (session) => {
 export const getHistory = (session) => {
     return [...session.history];
 };
+/**
+ * Regex pattern for SUGGEST_FETCH marker (bounded quantifier to prevent ReDoS)
+ */
+export const SUGGEST_FETCH_PATTERN = /\[SUGGEST_FETCH:\s*([^\]]{1,2000})\]/;
+export const SUGGEST_FETCH_PATTERN_GLOBAL = /\[SUGGEST_FETCH:\s*[^\]]{1,2000}\]/g;
+/**
+ * Parse suggested fetch URL from AI response
+ */
+export const parseSuggestedFetch = (response) => {
+    const match = response.match(SUGGEST_FETCH_PATTERN);
+    return match ? match[1].trim() : null;
+};
+/**
+ * Remove SUGGEST_FETCH markers from response
+ */
+export const removeSuggestFetchMarkers = (response) => {
+    return response.replace(SUGGEST_FETCH_PATTERN_GLOBAL, "").trim();
+};
+/**
+ * Get available references from script
+ */
+export const getReferences = (script) => {
+    return script.scriptMeta?.references || [];
+};
+/**
+ * Fetch reference content by URL
+ */
+export const fetchReference = async (url, verbose = false) => {
+    return fetchUrlContent(url, 8000, verbose);
+};
+/**
+ * Find matching reference for a query
+ */
+export const findReference = (script, query) => {
+    const references = getReferences(script);
+    return findMatchingReference(references, query);
+};
+/**
+ * Send a question with fetched reference content
+ */
+export const sendInteractiveQueryWithFetch = async (filteredScript, question, fetchedContent, session, options) => {
+    if (filteredScript.beats.length === 0) {
+        return "No content available to answer the question.";
+    }
+    // Build system prompt for fetched content mode
+    let systemPrompt = DEFAULT_INTERACTIVE_SYSTEM_PROMPT_WITH_FETCH;
+    if (options.lang) {
+        const langName = getLanguageName(options.lang);
+        systemPrompt = `${systemPrompt}\n- IMPORTANT: Write the answer in ${langName}`;
+    }
+    // Build user prompt with fetched content
+    const baseUserPrompt = buildInteractiveUserPrompt(filteredScript, question, session.history);
+    // Insert fetched content before the question
+    const fetchedSection = [
+        "",
+        "---",
+        "Additional reference content fetched from URL:",
+        `URL: ${fetchedContent.url}`,
+        fetchedContent.title ? `Title: ${fetchedContent.title}` : "",
+        "",
+        fetchedContent.content,
+        "---",
+        "",
+    ]
+        .filter(Boolean)
+        .join("\n");
+    // Insert before "Current question:" or at the end
+    const insertPoint = baseUserPrompt.indexOf("Current question:");
+    const userPrompt = insertPoint >= 0 ? baseUserPrompt.slice(0, insertPoint) + fetchedSection + baseUserPrompt.slice(insertPoint) : baseUserPrompt + fetchedSection;
+    const answer = await executeLLM(systemPrompt, userPrompt, options, options.verbose ? `Interactive query with fetch: ${question}` : undefined);
+    // Add to history (include note about fetched content)
+    session.history.push({ role: "user", content: `${question} (with reference: ${fetchedContent.url})` });
+    session.history.push({ role: "assistant", content: answer });
+    return answer;
+};

package/lib/core/ai/command/query/prompts.d.ts

@@ -15,7 +15,11 @@ export declare const buildUserPrompt: (script: ExtendedScript, question: string)
 /**
  * Default system prompt for interactive query
  */
-export declare const DEFAULT_INTERACTIVE_SYSTEM_PROMPT = "You are answering questions based on the content provided.\n- Answer based ONLY on the information in the provided content\n- If the answer cannot be found in the content, say so clearly\n- Be concise and direct in your answers\n- Do not make up information that is not in the content\n- You may reference previous conversation when answering follow-up questions";
+export declare const DEFAULT_INTERACTIVE_SYSTEM_PROMPT = "You are answering questions based on the content provided.\n- Answer based ONLY on the information in the provided content\n- If the answer cannot be found in the content, say so clearly\n- Be concise and direct in your answers\n- Do not make up information that is not in the content\n- You may reference previous conversation when answering follow-up questions\n- If references are available and the user asks for more details, mention which reference could provide more information\n- When you suggest fetching a reference for more details, include [SUGGEST_FETCH: <url>] in your response";
+/**
+ * Default system prompt for interactive query with fetched content
+ */
+export declare const DEFAULT_INTERACTIVE_SYSTEM_PROMPT_WITH_FETCH = "You are answering questions based on the content provided, including fetched reference content.\n- Answer based on both the main content and any fetched reference content\n- If the answer cannot be found, say so clearly\n- Be concise and direct in your answers\n- Do not make up information\n- You may reference previous conversation when answering follow-up questions\n- Prioritize information from fetched content when it's more detailed and relevant";
 /**
  * Get system prompt for interactive mode
  */

package/lib/core/ai/command/query/prompts.js

@@ -44,7 +44,19 @@ export const DEFAULT_INTERACTIVE_SYSTEM_PROMPT = `You are answering questions ba
 - If the answer cannot be found in the content, say so clearly
 - Be concise and direct in your answers
 - Do not make up information that is not in the content
-- You may reference previous conversation when answering follow-up questions`;
+- You may reference previous conversation when answering follow-up questions
+- If references are available and the user asks for more details, mention which reference could provide more information
+- When you suggest fetching a reference for more details, include [SUGGEST_FETCH: <url>] in your response`;
+/**
+ * Default system prompt for interactive query with fetched content
+ */
+export const DEFAULT_INTERACTIVE_SYSTEM_PROMPT_WITH_FETCH = `You are answering questions based on the content provided, including fetched reference content.
+- Answer based on both the main content and any fetched reference content
+- If the answer cannot be found, say so clearly
+- Be concise and direct in your answers
+- Do not make up information
+- You may reference previous conversation when answering follow-up questions
+- Prioritize information from fetched content when it's more detailed and relevant`;
 /**
  * Get system prompt for interactive mode
  */

package/lib/core/ai/llm.js

@@ -103,31 +103,120 @@ export const getLanguageName = (langCode) => {
     };
     return langMap[langCode] || langCode;
 };
+/**
+ * Build beat content including metadata
+ */
+const buildBeatContent = (beat, index) => {
+    const lines = [];
+    // Main text
+    const text = beat.text || "";
+    if (!text.trim())
+        return "";
+    lines.push(`[${index}] ${text}`);
+    // Add metadata if available
+    const meta = beat.meta;
+    if (meta) {
+        // Tags for categorization
+        if (meta.tags && meta.tags.length > 0) {
+            lines.push(`  Tags: ${meta.tags.join(", ")}`);
+        }
+        // Context provides additional information not in the text
+        if (meta.context) {
+            lines.push(`  Context: ${meta.context}`);
+        }
+        // Keywords highlight important terms
+        if (meta.keywords && meta.keywords.length > 0) {
+            lines.push(`  Keywords: ${meta.keywords.join(", ")}`);
+        }
+        // Expected questions this beat can answer
+        if (meta.expectedQuestions && meta.expectedQuestions.length > 0) {
+            lines.push(`  Can answer: ${meta.expectedQuestions.join("; ")}`);
+        }
+    }
+    return lines.join("\n");
+};
+/**
+ * Build script-level metadata section
+ */
+const buildScriptMetaContent = (script) => {
+    const meta = script.scriptMeta;
+    if (!meta)
+        return "";
+    const lines = [];
+    // Background info
+    if (meta.background) {
+        lines.push(`Background: ${meta.background}`);
+    }
+    // Audience and prerequisites
+    if (meta.audience) {
+        lines.push(`Target audience: ${meta.audience}`);
+    }
+    if (meta.prerequisites && meta.prerequisites.length > 0) {
+        lines.push(`Prerequisites: ${meta.prerequisites.join(", ")}`);
+    }
+    // Goals
+    if (meta.goals && meta.goals.length > 0) {
+        lines.push(`Goals: ${meta.goals.join("; ")}`);
+    }
+    // Keywords
+    if (meta.keywords && meta.keywords.length > 0) {
+        lines.push(`Keywords: ${meta.keywords.join(", ")}`);
+    }
+    // References
+    if (meta.references && meta.references.length > 0) {
+        lines.push("References:");
+        meta.references.forEach((ref) => {
+            const title = ref.title || ref.url;
+            const desc = ref.description ? ` - ${ref.description}` : "";
+            lines.push(`  - [${ref.type || "web"}] ${title}: ${ref.url}${desc}`);
+        });
+    }
+    // FAQ
+    if (meta.faq && meta.faq.length > 0) {
+        lines.push("FAQ:");
+        meta.faq.forEach((faq) => {
+            lines.push(`  Q: ${faq.question}`);
+            lines.push(`  A: ${faq.answer}`);
+        });
+    }
+    // Author info
+    if (meta.author) {
+        lines.push(`Author: ${meta.author}`);
+    }
+    return lines.length > 0 ? lines.join("\n") : "";
+};
 /**
  * Build script content for user prompt (common part)
  */
 export const buildScriptContent = (script) => {
     const parts = [];
-    // Add script metadata
+    // Add script title and language
     parts.push(`# Script: ${script.title}`);
     parts.push(`Language: ${script.lang}`);
     parts.push("");
-    // Collect all text from beats grouped by section
+    // Add script-level metadata
+    const scriptMetaContent = buildScriptMetaContent(script);
+    if (scriptMetaContent) {
+        parts.push("## About this content");
+        parts.push(scriptMetaContent);
+        parts.push("");
+    }
+    // Collect all content from beats grouped by section
     const sections = new Map();
     script.beats.forEach((beat, index) => {
-        const text = beat.text || "";
-        if (!text.trim())
+        const content = buildBeatContent(beat, index);
+        if (!content)
             return;
         const section = beat.meta?.section || "main";
         if (!sections.has(section)) {
             sections.set(section, []);
         }
-        sections.get(section).push(`[${index}] ${text}`);
+        sections.get(section).push(content);
     });
     // Output by section
-    sections.forEach((texts, section) => {
+    sections.forEach((contents, section) => {
         parts.push(`## Section: ${section}`);
-        texts.forEach((t) => parts.push(t));
+        contents.forEach((c) => parts.push(c));
         parts.push("");
     });
     return parts.join("\n");

package/lib/core/ai/utils/fetcher.d.ts

@@ -0,0 +1,25 @@
+/**
+ * Fetched content result
+ */
+export interface FetchedContent {
+    url: string;
+    title: string | null;
+    content: string;
+    error?: string;
+}
+/**
+ * Fetch URL content and extract text
+ */
+export declare const fetchUrlContent: (url: string, maxLength?: number, verbose?: boolean) => Promise<FetchedContent>;
+/**
+ * Find matching reference URL from script metadata
+ */
+export declare const findMatchingReference: (references: Array<{
+    url: string;
+    title?: string;
+    description?: string;
+}> | undefined, query: string) => {
+    url: string;
+    title?: string;
+    description?: string;
+} | null;

package/lib/core/ai/utils/fetcher.js

@@ -0,0 +1,122 @@
+import { GraphAILogger } from "graphai";
+/**
+ * Strip HTML tags and extract text content
+ */
+const stripHtml = (html) => {
+    // Remove script and style elements
+    let text = html.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, "");
+    text = text.replace(/<style[^>]*>[\s\S]*?<\/style>/gi, "");
+    // Remove HTML comments
+    text = text.replace(/<!--[\s\S]*?-->/g, "");
+    // Replace common block elements with newlines
+    text = text.replace(/<\/(p|div|h[1-6]|li|tr|br)[^>]*>/gi, "\n");
+    // Remove all remaining HTML tags
+    // eslint-disable-next-line sonarjs/slow-regex -- standard HTML tag removal pattern, safe for typical HTML
+    text = text.replace(/<[^>]*>/g, " ");
+    // Decode common HTML entities
+    text = text.replace(/&nbsp;/g, " ");
+    text = text.replace(/&amp;/g, "&");
+    text = text.replace(/&lt;/g, "<");
+    text = text.replace(/&gt;/g, ">");
+    text = text.replace(/&quot;/g, '"');
+    text = text.replace(/&#39;/g, "'");
+    // Normalize whitespace
+    text = text.replace(/\s+/g, " ");
+    text = text.replace(/\n\s*\n/g, "\n\n");
+    return text.trim();
+};
+/**
+ * Extract title from HTML
+ */
+const extractTitle = (html) => {
+    const match = html.match(/<title[^>]*>([^<]+)<\/title>/i);
+    return match ? match[1].trim() : null;
+};
+/**
+ * Fetch URL content and extract text
+ */
+export const fetchUrlContent = async (url, maxLength = 8000, verbose = false) => {
+    try {
+        if (verbose) {
+            GraphAILogger.info(`Fetching URL: ${url}`);
+        }
+        const controller = new AbortController();
+        const timeoutId = setTimeout(() => controller.abort(), 30000);
+        const response = await fetch(url, {
+            signal: controller.signal,
+            headers: {
+                "User-Agent": "Mozilla/5.0 (compatible; MulmoCast/1.0; +https://github.com/receptron/mulmocast)",
+                Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+            },
+        });
+        clearTimeout(timeoutId);
+        if (!response.ok) {
+            return {
+                url,
+                title: null,
+                content: "",
+                error: `HTTP ${response.status}: ${response.statusText}`,
+            };
+        }
+        const contentType = response.headers.get("content-type") || "";
+        // Handle non-HTML content
+        if (!contentType.includes("text/html") && !contentType.includes("application/xhtml+xml")) {
+            if (contentType.includes("text/plain")) {
+                const text = await response.text();
+                return {
+                    url,
+                    title: null,
+                    content: text.substring(0, maxLength),
+                };
+            }
+            return {
+                url,
+                title: null,
+                content: "",
+                error: `Unsupported content type: ${contentType}`,
+            };
+        }
+        const html = await response.text();
+        const title = extractTitle(html);
+        const content = stripHtml(html);
+        // Truncate if needed
+        const truncatedContent = content.length > maxLength ? content.substring(0, maxLength) + "..." : content;
+        if (verbose) {
+            GraphAILogger.info(`Fetched ${content.length} chars from ${url}`);
+        }
+        return {
+            url,
+            title,
+            content: truncatedContent,
+        };
+    }
+    catch (error) {
+        const errorMessage = error instanceof Error ? error.message : String(error);
+        return {
+            url,
+            title: null,
+            content: "",
+            error: errorMessage,
+        };
+    }
+};
+/**
+ * Find matching reference URL from script metadata
+ */
+export const findMatchingReference = (references, query) => {
+    if (!references || references.length === 0) {
+        return null;
+    }
+    const lowerQuery = query.toLowerCase();
+    // Try to find a reference that matches keywords in the query
+    for (const ref of references) {
+        const refText = [ref.title, ref.description, ref.url].filter(Boolean).join(" ").toLowerCase();
+        // Simple keyword matching
+        const queryWords = lowerQuery.split(/\s+/).filter((w) => w.length > 2);
+        const matchScore = queryWords.filter((word) => refText.includes(word)).length;
+        if (matchScore >= 2 || (queryWords.length === 1 && matchScore === 1)) {
+            return ref;
+        }
+    }
+    return null;
+};

package/lib/index.d.ts

@@ -4,10 +4,12 @@ export { filterBySection, filterByTags, stripExtendedFields } from "./core/prepr
 export { listProfiles } from "./core/preprocessing/profiles.js";
 export { summarizeScript } from "./core/ai/command/summarize/index.js";
 export { queryScript } from "./core/ai/command/query/index.js";
-export { createInteractiveSession, sendInteractiveQuery, clearHistory, getHistory } from "./core/ai/command/query/interactive.js";
-export type { BeatVariant, BeatMeta, ExtendedBeat, ExtendedScript, OutputProfile, ProcessOptions, ProfileInfo } from "./types/index.js";
+export { createInteractiveSession, sendInteractiveQuery, sendInteractiveQueryWithFetch, clearHistory, getHistory, getReferences, findReference, fetchReference, parseSuggestedFetch, removeSuggestFetchMarkers, } from "./core/ai/command/query/interactive.js";
+export { fetchUrlContent } from "./core/ai/utils/fetcher.js";
+export type { FetchedContent } from "./core/ai/utils/fetcher.js";
+export type { BeatVariant, BeatMeta, ExtendedBeat, ExtendedScript, OutputProfile, ProcessOptions, ProfileInfo, Reference, FAQ, ScriptMeta, } from "./types/index.js";
 export type { SummarizeOptions, SummarizeResult, LLMProvider, SummarizeFormat, ProviderConfig } from "./types/summarize.js";
 export type { QueryOptions, QueryResult, ConversationMessage, InteractiveQuerySession } from "./types/query.js";
-export { beatVariantSchema, beatMetaSchema, extendedBeatSchema, extendedScriptSchema, outputProfileSchema } from "./types/index.js";
+export { beatVariantSchema, beatMetaSchema, extendedBeatSchema, extendedScriptSchema, outputProfileSchema, referenceSchema, faqSchema, scriptMetaSchema, } from "./types/index.js";
 export { summarizeOptionsSchema, llmProviderSchema, summarizeFormatSchema } from "./types/summarize.js";
 export { queryOptionsSchema } from "./types/query.js";

package/lib/index.js

@@ -6,8 +6,10 @@ export { listProfiles } from "./core/preprocessing/profiles.js";
 // AI API
 export { summarizeScript } from "./core/ai/command/summarize/index.js";
 export { queryScript } from "./core/ai/command/query/index.js";
-export { createInteractiveSession, sendInteractiveQuery, clearHistory, getHistory } from "./core/ai/command/query/interactive.js";
+export { createInteractiveSession, sendInteractiveQuery, sendInteractiveQueryWithFetch, clearHistory, getHistory, getReferences, findReference, fetchReference, parseSuggestedFetch, removeSuggestFetchMarkers, } from "./core/ai/command/query/interactive.js";
+// Utilities
+export { fetchUrlContent } from "./core/ai/utils/fetcher.js";
 // Schemas (for validation)
-export { beatVariantSchema, beatMetaSchema, extendedBeatSchema, extendedScriptSchema, outputProfileSchema } from "./types/index.js";
+export { beatVariantSchema, beatMetaSchema, extendedBeatSchema, extendedScriptSchema, outputProfileSchema, referenceSchema, faqSchema, scriptMetaSchema, } from "./types/index.js";
 export { summarizeOptionsSchema, llmProviderSchema, summarizeFormatSchema } from "./types/summarize.js";
 export { queryOptionsSchema } from "./types/query.js";

package/lib/types/index.d.ts

@@ -860,6 +860,59 @@ export declare const outputProfileSchema: z.ZodObject<{
     description: z.ZodOptional<z.ZodString>;
 }, z.core.$strip>;
 export type OutputProfile = z.infer<typeof outputProfileSchema>;
+/**
+ * Reference - external resource reference
+ */
+export declare const referenceSchema: z.ZodObject<{
+    type: z.ZodOptional<z.ZodEnum<{
+        web: "web";
+        code: "code";
+        document: "document";
+        video: "video";
+    }>>;
+    url: z.ZodString;
+    title: z.ZodOptional<z.ZodString>;
+    description: z.ZodOptional<z.ZodString>;
+}, z.core.$strip>;
+export type Reference = z.infer<typeof referenceSchema>;
+/**
+ * FAQ - frequently asked question
+ */
+export declare const faqSchema: z.ZodObject<{
+    question: z.ZodString;
+    answer: z.ZodString;
+    relatedBeats: z.ZodOptional<z.ZodArray<z.ZodString>>;
+}, z.core.$strip>;
+export type FAQ = z.infer<typeof faqSchema>;
+/**
+ * Script Meta - script-level metadata for AI features
+ */
+export declare const scriptMetaSchema: z.ZodObject<{
+    audience: z.ZodOptional<z.ZodString>;
+    prerequisites: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    goals: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    background: z.ZodOptional<z.ZodString>;
+    faq: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        question: z.ZodString;
+        answer: z.ZodString;
+        relatedBeats: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    }, z.core.$strip>>>;
+    keywords: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    references: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        type: z.ZodOptional<z.ZodEnum<{
+            web: "web";
+            code: "code";
+            document: "document";
+            video: "video";
+        }>>;
+        url: z.ZodString;
+        title: z.ZodOptional<z.ZodString>;
+        description: z.ZodOptional<z.ZodString>;
+    }, z.core.$strip>>>;
+    author: z.ZodOptional<z.ZodString>;
+    version: z.ZodOptional<z.ZodString>;
+}, z.core.$strip>;
+export type ScriptMeta = z.infer<typeof scriptMetaSchema>;
 /**
  * Extended Script - script with variants, meta, and outputProfiles
  */
@@ -1924,6 +1977,31 @@ export declare const extendedScriptSchema: z.ZodObject<{
         name: z.ZodString;
         description: z.ZodOptional<z.ZodString>;
     }, z.core.$strip>>>;
+    scriptMeta: z.ZodOptional<z.ZodObject<{
+        audience: z.ZodOptional<z.ZodString>;
+        prerequisites: z.ZodOptional<z.ZodArray<z.ZodString>>;
+        goals: z.ZodOptional<z.ZodArray<z.ZodString>>;
+        background: z.ZodOptional<z.ZodString>;
+        faq: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            question: z.ZodString;
+            answer: z.ZodString;
+            relatedBeats: z.ZodOptional<z.ZodArray<z.ZodString>>;
+        }, z.core.$strip>>>;
+        keywords: z.ZodOptional<z.ZodArray<z.ZodString>>;
+        references: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            type: z.ZodOptional<z.ZodEnum<{
+                web: "web";
+                code: "code";
+                document: "document";
+                video: "video";
+            }>>;
+            url: z.ZodString;
+            title: z.ZodOptional<z.ZodString>;
+            description: z.ZodOptional<z.ZodString>;
+        }, z.core.$strip>>>;
+        author: z.ZodOptional<z.ZodString>;
+        version: z.ZodOptional<z.ZodString>;
+    }, z.core.$strip>>;
 }, z.core.$strict>;
 export type ExtendedScript = z.infer<typeof extendedScriptSchema>;
 /**

package/lib/types/index.js

@@ -33,10 +33,47 @@ export const outputProfileSchema = z.object({
     name: z.string(),
     description: z.string().optional(),
 });
+/**
+ * Reference - external resource reference
+ */
+export const referenceSchema = z.object({
+    type: z.enum(["web", "code", "document", "video"]).optional(),
+    url: z.string(),
+    title: z.string().optional(),
+    description: z.string().optional(),
+});
+/**
+ * FAQ - frequently asked question
+ */
+export const faqSchema = z.object({
+    question: z.string(),
+    answer: z.string(),
+    relatedBeats: z.array(z.string()).optional(),
+});
+/**
+ * Script Meta - script-level metadata for AI features
+ */
+export const scriptMetaSchema = z.object({
+    // Target audience and prerequisites
+    audience: z.string().optional(),
+    prerequisites: z.array(z.string()).optional(),
+    // Learning goals and background
+    goals: z.array(z.string()).optional(),
+    background: z.string().optional(),
+    // FAQ for quick Q&A matching
+    faq: z.array(faqSchema).optional(),
+    // Search and discovery
+    keywords: z.array(z.string()).optional(),
+    references: z.array(referenceSchema).optional(),
+    // Authoring info
+    author: z.string().optional(),
+    version: z.string().optional(),
+});
 /**
  * Extended Script - script with variants, meta, and outputProfiles
  */
 export const extendedScriptSchema = mulmoScriptSchema.extend({
     beats: z.array(extendedBeatSchema),
     outputProfiles: z.record(z.string(), outputProfileSchema).optional(),
+    scriptMeta: scriptMetaSchema.optional(),
 });

package/lib/types/query.d.ts

@@ -1,4 +1,5 @@
 import { z } from "zod";
+import type { FetchedContent } from "../core/ai/utils/fetcher.js";
 /**
  * Query Options - configuration for querying script content
  */
@@ -42,4 +43,5 @@ export interface InteractiveQuerySession {
     scriptTitle: string;
     beatCount: number;
     history: ConversationMessage[];
+    fetchedContent?: FetchedContent;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mulmocast-preprocessor",
-  "version": "0.2.1",
+  "version": "0.3.0",
   "description": "Preprocessor for MulmoScript",
   "type": "module",
   "main": "lib/index.js",