mulmocast-preprocessor 0.4.0 → 0.5.0

package/README.md

@@ -106,9 +106,9 @@ mulmocast-preprocessor query script.json  # Omit question for interactive mode
 
 ```typescript
 import { processScript, listProfiles, applyProfile } from "mulmocast-preprocessor";
-import type { ExtendedScript } from "mulmocast-preprocessor";
+import type { ExtendedMulmoScript } from "mulmocast-preprocessor";
 
-const script: ExtendedScript = {
+const script: ExtendedMulmoScript = {
   title: "My Presentation",
   beats: [
     {
@@ -148,7 +148,7 @@ const result = processScript(script, {
 Main processing function that applies profile and filters.
 
 **Parameters:**
-- `script: ExtendedScript` - Input script with variants/meta
+- `script: ExtendedMulmoScript` - Input script with variants/meta
 - `options: ProcessOptions` - Processing options
   - `profile?: string` - Profile name to apply
   - `section?: string` - Filter by section
@@ -161,7 +161,7 @@ Main processing function that applies profile and filters.
 Apply a profile to the script, replacing text/image and skipping marked beats.
 
 **Parameters:**
-- `script: ExtendedScript` - Input script
+- `script: ExtendedMulmoScript` - Input script
 - `profileName: string` - Profile name
 
 **Returns:** `MulmoScript` - Processed script
@@ -171,7 +171,7 @@ Apply a profile to the script, replacing text/image and skipping marked beats.
 Get list of available profiles from script.
 
 **Parameters:**
-- `script: ExtendedScript` - Input script
+- `script: ExtendedMulmoScript` - Input script
 
 **Returns:** `ProfileInfo[]` - Array of profile info with beat counts
 
@@ -188,7 +188,7 @@ Filter beats by tags (extracts beats that have any of the specified tags).
 Generate a summary of the script content using LLM.
 
 **Parameters:**
-- `script: ExtendedScript` - Input script
+- `script: ExtendedMulmoScript` - Input script
 - `options: SummarizeOptions` - Summarization options
   - `provider?: LLMProvider` - LLM provider (default: "openai")
   - `model?: string` - Model name
@@ -204,7 +204,7 @@ Generate a summary of the script content using LLM.
 Ask a question about the script content.
 
 **Parameters:**
-- `script: ExtendedScript` - Input script
+- `script: ExtendedMulmoScript` - Input script
 - `question: string` - Question to ask
 - `options: QueryOptions` - Query options (same as summarize)
 
@@ -215,7 +215,7 @@ Ask a question about the script content.
 Create an interactive query session for follow-up questions.
 
 **Parameters:**
-- `script: ExtendedScript` - Input script
+- `script: ExtendedMulmoScript` - Input script
 - `options: QueryOptions` - Query options
 
 **Returns:** Session object with `sendInteractiveQuery()` method
@@ -233,10 +233,10 @@ For AI features (summarize, query), set the API key for your LLM provider:
 
 ## Extended Schema
 
-### ExtendedBeat
+### ExtendedMulmoBeat
 
 ```typescript
-interface ExtendedBeat extends MulmoBeat {
+interface ExtendedMulmoBeat extends MulmoBeat {
   variants?: Record<string, BeatVariant>;
   meta?: BeatMeta;
 }

package/lib/cli/utils.d.ts

@@ -1,4 +1,4 @@
-import type { ExtendedScript } from "@mulmocast/extended-types";
+import type { ExtendedMulmoScript } from "@mulmocast/extended-types";
 /**
  * Check if input is a URL
  */
@@ -6,4 +6,4 @@ export declare const isUrl: (input: string) => boolean;
 /**
  * Load script from file path or URL
  */
-export declare const loadScript: (input: string) => Promise<ExtendedScript>;
+export declare const loadScript: (input: string) => Promise<ExtendedMulmoScript>;

package/lib/core/ai/command/query/index.d.ts

@@ -1,8 +1,8 @@
-import type { ExtendedScript } from "@mulmocast/extended-types";
+import type { ExtendedMulmoScript } from "@mulmocast/extended-types";
 import type { QueryOptions, QueryResult } from "../../../../types/query.js";
 /**
  * Main query function - answers a question based on script content
  */
-export declare const queryScript: (script: ExtendedScript, question: string, options?: Partial<QueryOptions>) => Promise<QueryResult>;
+export declare const queryScript: (script: ExtendedMulmoScript, question: string, options?: Partial<QueryOptions>) => Promise<QueryResult>;
 export type { QueryOptions, QueryResult } from "../../../../types/query.js";
 export { queryOptionsSchema } from "../../../../types/query.js";

package/lib/core/ai/command/query/interactive.d.ts

@@ -1,18 +1,18 @@
-import type { ExtendedScript, Reference } from "@mulmocast/extended-types";
+import type { ExtendedMulmoScript, Reference } from "@mulmocast/extended-types";
 import type { QueryOptions, InteractiveQuerySession, ConversationMessage } from "../../../../types/query.js";
 import { type FetchedContent } from "../../utils/fetcher.js";
 /**
  * Create an interactive query session
  */
-export declare const createInteractiveSession: (script: ExtendedScript, options?: Partial<QueryOptions>) => {
+export declare const createInteractiveSession: (script: ExtendedMulmoScript, options?: Partial<QueryOptions>) => {
     session: InteractiveQuerySession;
-    filteredScript: ExtendedScript;
+    filteredScript: ExtendedMulmoScript;
     validatedOptions: QueryOptions;
 };
 /**
  * Send a question in an interactive session
  */
-export declare const sendInteractiveQuery: (filteredScript: ExtendedScript, question: string, session: InteractiveQuerySession, options: QueryOptions) => Promise<string>;
+export declare const sendInteractiveQuery: (filteredScript: ExtendedMulmoScript, question: string, session: InteractiveQuerySession, options: QueryOptions) => Promise<string>;
 /**
  * Clear conversation history
  */
@@ -37,7 +37,7 @@ export declare const removeSuggestFetchMarkers: (response: string) => string;
 /**
  * Get available references from script
  */
-export declare const getReferences: (script: ExtendedScript) => Reference[];
+export declare const getReferences: (script: ExtendedMulmoScript) => Reference[];
 /**
  * Fetch reference content by URL
  */
@@ -45,8 +45,8 @@ export declare const fetchReference: (url: string, verbose?: boolean) => Promise
 /**
  * Find matching reference for a query
  */
-export declare const findReference: (script: ExtendedScript, query: string) => Reference | null;
+export declare const findReference: (script: ExtendedMulmoScript, query: string) => Reference | null;
 /**
  * Send a question with fetched reference content
  */
-export declare const sendInteractiveQueryWithFetch: (filteredScript: ExtendedScript, question: string, fetchedContent: FetchedContent, session: InteractiveQuerySession, options: QueryOptions) => Promise<string>;
+export declare const sendInteractiveQueryWithFetch: (filteredScript: ExtendedMulmoScript, question: string, fetchedContent: FetchedContent, session: InteractiveQuerySession, options: QueryOptions) => Promise<string>;

package/lib/core/ai/command/query/prompts.d.ts

@@ -1,5 +1,5 @@
 import type { QueryOptions, ConversationMessage } from "../../../../types/query.js";
-import type { ExtendedScript } from "@mulmocast/extended-types";
+import type { ExtendedMulmoScript } from "@mulmocast/extended-types";
 /**
  * Default system prompt for query
  */
@@ -11,7 +11,7 @@ export declare const getSystemPrompt: (options: QueryOptions) => string;
 /**
  * Build user prompt from script and question
  */
-export declare const buildUserPrompt: (script: ExtendedScript, question: string) => string;
+export declare const buildUserPrompt: (script: ExtendedMulmoScript, question: string) => string;
 /**
  * Default system prompt for interactive query
  */
@@ -27,4 +27,4 @@ export declare const getInteractiveSystemPrompt: (options: QueryOptions) => stri
 /**
  * Build user prompt with conversation history for interactive mode
  */
-export declare const buildInteractiveUserPrompt: (script: ExtendedScript, question: string, history: ConversationMessage[]) => string;
+export declare const buildInteractiveUserPrompt: (script: ExtendedMulmoScript, question: string, history: ConversationMessage[]) => string;

package/lib/core/ai/command/summarize/index.d.ts

@@ -1,8 +1,8 @@
-import type { ExtendedScript } from "@mulmocast/extended-types";
+import type { ExtendedMulmoScript } from "@mulmocast/extended-types";
 import type { SummarizeOptions, SummarizeResult } from "../../../../types/summarize.js";
 /**
  * Main summarize function - generates a summary of the entire script
  */
-export declare const summarizeScript: (script: ExtendedScript, options?: Partial<SummarizeOptions>) => Promise<SummarizeResult>;
+export declare const summarizeScript: (script: ExtendedMulmoScript, options?: Partial<SummarizeOptions>) => Promise<SummarizeResult>;
 export type { SummarizeOptions, SummarizeResult, LLMProvider, SummarizeFormat } from "../../../../types/summarize.js";
 export { summarizeOptionsSchema, llmProviderSchema, summarizeFormatSchema } from "../../../../types/summarize.js";

package/lib/core/ai/command/summarize/prompts.d.ts

@@ -1,5 +1,5 @@
 import type { SummarizeOptions } from "../../../../types/summarize.js";
-import type { ExtendedScript } from "@mulmocast/extended-types";
+import type { ExtendedMulmoScript } from "@mulmocast/extended-types";
 /**
  * Default system prompt for text summary
  */
@@ -11,7 +11,7 @@ export declare const DEFAULT_SYSTEM_PROMPT_MARKDOWN = "You are creating a summar
 /**
  * Build user prompt from entire script
  */
-export declare const buildUserPrompt: (script: ExtendedScript, options: SummarizeOptions) => string;
+export declare const buildUserPrompt: (script: ExtendedMulmoScript, options: SummarizeOptions) => string;
 /**
  * Get system prompt based on format and language
  */

package/lib/core/ai/llm.d.ts

@@ -1,4 +1,4 @@
-import type { ExtendedScript } from "@mulmocast/extended-types";
+import type { ExtendedMulmoScript } from "@mulmocast/extended-types";
 import type { LLMProvider } from "../../types/summarize.js";
 /**
  * Base options for LLM operations
@@ -34,7 +34,7 @@ export declare const getProviderApiKey: (provider: LLMProvider) => string | unde
 /**
  * Filter script based on options (section, tags)
  */
-export declare const filterScript: (script: ExtendedScript, options: BaseLLMOptions) => ExtendedScript;
+export declare const filterScript: (script: ExtendedMulmoScript, options: BaseLLMOptions) => ExtendedMulmoScript;
 /**
  * Get language name from code
  */
@@ -42,7 +42,7 @@ export declare const getLanguageName: (langCode: string) => string;
 /**
  * Build script content for user prompt (common part)
  */
-export declare const buildScriptContent: (script: ExtendedScript) => string;
+export declare const buildScriptContent: (script: ExtendedMulmoScript) => string;
 /**
  * Command execution result
  */
@@ -54,7 +54,7 @@ export interface CommandResult {
 /**
  * Execute a command (summarize, query, etc.) with common logic
  */
-export declare const executeCommand: <T extends BaseLLMOptions>(script: ExtendedScript, options: T, getSystemPrompt: (opts: T) => string, buildUserPrompt: (script: ExtendedScript) => string, verboseMessage: string) => Promise<CommandResult | null>;
+export declare const executeCommand: <T extends BaseLLMOptions>(script: ExtendedMulmoScript, options: T, getSystemPrompt: (opts: T) => string, buildUserPrompt: (script: ExtendedMulmoScript) => string, verboseMessage: string) => Promise<CommandResult | null>;
 /**
  * Execute LLM call with GraphAI
  */

package/lib/core/ai/utils/fetcher.d.ts

@@ -7,6 +7,14 @@ export interface FetchedContent {
     content: string;
     error?: string;
 }
+/**
+ * Strip HTML tags and extract text content
+ */
+export declare const stripHtml: (html: string) => string;
+/**
+ * Extract title from HTML using indexOf (avoids regex ReDoS)
+ */
+export declare const extractTitle: (html: string) => string | null;
 /**
  * Fetch URL content and extract text
  */

package/lib/core/ai/utils/fetcher.js

@@ -1,36 +1,87 @@
 import { GraphAILogger } from "graphai";
+/**
+ * Remove all occurrences of a tag block (e.g. <script>...</script>) using indexOf
+ */
+const removeTagBlock = (html, openTag, closeTag) => {
+    let result = html;
+    let lower = result.toLowerCase();
+    let start = lower.indexOf(openTag);
+    while (start !== -1) {
+        const end = lower.indexOf(closeTag, start);
+        if (end === -1)
+            break;
+        result = result.substring(0, start) + result.substring(end + closeTag.length);
+        lower = result.toLowerCase();
+        start = lower.indexOf(openTag);
+    }
+    return result;
+};
+const BLOCK_ELEMENTS = new Set(["p", "div", "h1", "h2", "h3", "h4", "h5", "h6", "li", "tr", "br"]);
+/**
+ * Remove all HTML tags using indexOf, replacing block closings with newlines
+ */
+const removeTags = (html) => {
+    const parts = [];
+    let i = 0;
+    while (i < html.length) {
+        const tagStart = html.indexOf("<", i);
+        if (tagStart === -1) {
+            parts.push(html.substring(i));
+            break;
+        }
+        parts.push(html.substring(i, tagStart));
+        const tagEnd = html.indexOf(">", tagStart);
+        if (tagEnd === -1) {
+            parts.push(html.substring(tagStart));
+            break;
+        }
+        const tagContent = html.substring(tagStart + 1, tagEnd);
+        const closeMatch = tagContent.match(/^\/(\w+)/);
+        if (closeMatch && BLOCK_ELEMENTS.has(closeMatch[1].toLowerCase())) {
+            parts.push("\n");
+        }
+        i = tagEnd + 1;
+    }
+    return parts.join("");
+};
 /**
  * Strip HTML tags and extract text content
  */
-const stripHtml = (html) => {
-    // Remove script and style elements
-    let text = html.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, "");
-    text = text.replace(/<style[^>]*>[\s\S]*?<\/style>/gi, "");
-    // Remove HTML comments
-    text = text.replace(/<!--[\s\S]*?-->/g, "");
-    // Replace common block elements with newlines
-    text = text.replace(/<\/(p|div|h[1-6]|li|tr|br)[^>]*>/gi, "\n");
-    // Remove all remaining HTML tags
-    // eslint-disable-next-line sonarjs/slow-regex -- standard HTML tag removal pattern, safe for typical HTML
-    text = text.replace(/<[^>]*>/g, " ");
-    // Decode common HTML entities
+export const stripHtml = (html) => {
+    // Remove script, style elements and comments using indexOf (avoids regex ReDoS)
+    let text = removeTagBlock(html, "<script", "</script>");
+    text = removeTagBlock(text, "<style", "</style>");
+    text = removeTagBlock(text, "<!--", "-->");
+    // Remove all HTML tags
+    text = removeTags(text);
+    // Decode common HTML entities (&amp; must be last to prevent double-decoding)
     text = text.replace(/&nbsp;/g, " ");
-    text = text.replace(/&amp;/g, "&");
     text = text.replace(/&lt;/g, "<");
     text = text.replace(/&gt;/g, ">");
     text = text.replace(/&quot;/g, '"');
     text = text.replace(/&#39;/g, "'");
+    text = text.replace(/&amp;/g, "&");
     // Normalize whitespace
     text = text.replace(/\s+/g, " ");
     text = text.replace(/\n\s*\n/g, "\n\n");
     return text.trim();
 };
 /**
- * Extract title from HTML
+ * Extract title from HTML using indexOf (avoids regex ReDoS)
  */
-const extractTitle = (html) => {
-    const match = html.match(/<title[^>]*>([^<]+)<\/title>/i);
-    return match ? match[1].trim() : null;
+export const extractTitle = (html) => {
+    const lower = html.toLowerCase();
+    const titleStart = lower.indexOf("<title");
+    if (titleStart === -1)
+        return null;
+    const contentStart = html.indexOf(">", titleStart);
+    if (contentStart === -1)
+        return null;
+    const titleEnd = lower.indexOf("</title>", contentStart);
+    if (titleEnd === -1)
+        return null;
+    const title = html.substring(contentStart + 1, titleEnd).trim();
+    return title || null;
 };
 /**
  * Fetch URL content and extract text

package/lib/core/preprocessing/filter.d.ts

@@ -1,14 +1,14 @@
 import type { MulmoScript } from "@mulmocast/types";
-import type { ExtendedScript } from "@mulmocast/extended-types";
+import type { ExtendedMulmoScript } from "@mulmocast/extended-types";
 /**
  * Filter beats by section (preserves meta for chaining)
  */
-export declare const filterBySection: (script: ExtendedScript, section: string) => ExtendedScript;
+export declare const filterBySection: (script: ExtendedMulmoScript, section: string) => ExtendedMulmoScript;
 /**
  * Filter beats by tags (preserves meta for chaining)
  */
-export declare const filterByTags: (script: ExtendedScript, tags: string[]) => ExtendedScript;
+export declare const filterByTags: (script: ExtendedMulmoScript, tags: string[]) => ExtendedMulmoScript;
 /**
  * Strip variants and meta fields, converting to standard MulmoScript
  */
-export declare const stripExtendedFields: (script: ExtendedScript) => MulmoScript;
+export declare const stripExtendedFields: (script: ExtendedMulmoScript) => MulmoScript;

package/lib/core/preprocessing/filter.js

@@ -3,7 +3,7 @@ const stripBeatExtendedFields = (beat) => {
     return baseBeat;
 };
 const filterBeatsToMulmoScript = (script, predicate) => {
-    const { outputProfiles: __outputProfiles, ...baseScript } = script;
+    const { outputProfiles: __outputProfiles, scriptMeta: __scriptMeta, ...baseScript } = script;
     return {
         ...baseScript,
         beats: script.beats.filter(predicate).map(stripBeatExtendedFields),

package/lib/core/preprocessing/process.d.ts

@@ -1,8 +1,8 @@
 import type { MulmoScript } from "@mulmocast/types";
-import type { ExtendedScript } from "@mulmocast/extended-types";
+import type { ExtendedMulmoScript } from "@mulmocast/extended-types";
 import type { ProcessOptions } from "../../types/index.js";
 /**
  * Main processing function
  * Applies filters first (while meta exists), then profile
  */
-export declare const processScript: (script: ExtendedScript, options?: ProcessOptions) => MulmoScript;
+export declare const processScript: (script: ExtendedMulmoScript, options?: ProcessOptions) => MulmoScript;

package/lib/core/preprocessing/profiles.d.ts

@@ -1,6 +1,6 @@
-import type { ExtendedScript } from "@mulmocast/extended-types";
+import type { ExtendedMulmoScript } from "@mulmocast/extended-types";
 import type { ProfileInfo } from "../../types/index.js";
 /**
  * Get list of available profiles from script
  */
-export declare const listProfiles: (script: ExtendedScript) => ProfileInfo[];
+export declare const listProfiles: (script: ExtendedMulmoScript) => ProfileInfo[];

package/lib/core/preprocessing/variant.d.ts

@@ -1,6 +1,6 @@
 import type { MulmoScript } from "@mulmocast/types";
-import type { ExtendedScript } from "@mulmocast/extended-types";
+import type { ExtendedMulmoScript } from "@mulmocast/extended-types";
 /**
  * Apply profile to script and return standard MulmoScript
  */
-export declare const applyProfile: (script: ExtendedScript, profileName: string) => MulmoScript;
+export declare const applyProfile: (script: ExtendedMulmoScript, profileName: string) => MulmoScript;

package/lib/core/preprocessing/variant.js

@@ -18,7 +18,7 @@ const applyVariantToBeat = (beat, profileName) => {
  * Apply profile to script and return standard MulmoScript
  */
 export const applyProfile = (script, profileName) => {
-    const { outputProfiles: __outputProfiles, ...baseScript } = script;
+    const { outputProfiles: __outputProfiles, scriptMeta: __scriptMeta, ...baseScript } = script;
     return {
         ...baseScript,
         beats: script.beats.map((beat) => applyVariantToBeat(beat, profileName)).filter((beat) => beat !== null),

package/lib/index.d.ts

@@ -5,10 +5,8 @@ export { listProfiles } from "./core/preprocessing/profiles.js";
 export { summarizeScript } from "./core/ai/command/summarize/index.js";
 export { queryScript } from "./core/ai/command/query/index.js";
 export { createInteractiveSession, sendInteractiveQuery, sendInteractiveQueryWithFetch, clearHistory, getHistory, getReferences, findReference, fetchReference, parseSuggestedFetch, removeSuggestFetchMarkers, } from "./core/ai/command/query/interactive.js";
-export { fetchUrlContent } from "./core/ai/utils/fetcher.js";
+export { fetchUrlContent, stripHtml, extractTitle } from "./core/ai/utils/fetcher.js";
 export type { FetchedContent } from "./core/ai/utils/fetcher.js";
-export type { BeatVariant, BeatMeta, ExtendedBeat, ExtendedScript, OutputProfile, Reference, FAQ, ScriptMeta, MulmoImageAsset, } from "@mulmocast/extended-types";
-export { beatVariantSchema, beatMetaSchema, extendedBeatSchema, extendedScriptSchema, outputProfileSchema, referenceSchema, faqSchema, scriptMetaSchema, } from "@mulmocast/extended-types";
 export type { ProcessOptions, ProfileInfo } from "./types/index.js";
 export type { SummarizeOptions, SummarizeResult, LLMProvider, SummarizeFormat, ProviderConfig } from "./types/summarize.js";
 export type { QueryOptions, QueryResult, ConversationMessage, InteractiveQuerySession } from "./types/query.js";

package/lib/index.js

@@ -8,8 +8,7 @@ export { summarizeScript } from "./core/ai/command/summarize/index.js";
 export { queryScript } from "./core/ai/command/query/index.js";
 export { createInteractiveSession, sendInteractiveQuery, sendInteractiveQueryWithFetch, clearHistory, getHistory, getReferences, findReference, fetchReference, parseSuggestedFetch, removeSuggestFetchMarkers, } from "./core/ai/command/query/interactive.js";
 // Utilities
-export { fetchUrlContent } from "./core/ai/utils/fetcher.js";
-export { beatVariantSchema, beatMetaSchema, extendedBeatSchema, extendedScriptSchema, outputProfileSchema, referenceSchema, faqSchema, scriptMetaSchema, } from "@mulmocast/extended-types";
+export { fetchUrlContent, stripHtml, extractTitle } from "./core/ai/utils/fetcher.js";
 // Schemas (local)
 export { summarizeOptionsSchema, llmProviderSchema, summarizeFormatSchema } from "./types/summarize.js";
 export { queryOptionsSchema } from "./types/query.js";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mulmocast-preprocessor",
-  "version": "0.4.0",
+  "version": "0.5.0",
   "description": "Preprocessor for MulmoScript",
   "type": "module",
   "main": "lib/index.js",
@@ -39,7 +39,7 @@
     "@graphai/groq_agent": "^2.0.2",
     "@graphai/openai_agent": "^2.0.9",
     "@graphai/vanilla": "^2.0.12",
-    "@mulmocast/extended-types": "^0.1.0",
+    "@mulmocast/extended-types": "^0.3.0",
     "@mulmocast/types": "^2.1.35",
     "dotenv": "^17.2.4",
     "graphai": "^2.0.16",