@synsci/cli-darwin-x64-baseline 1.1.71 → 1.1.72

package/bin/skills/markitdown/scripts/convert_literature.py

@@ -0,0 +1,262 @@
+#!/usr/bin/env python3
+"""
+Convert scientific literature PDFs to Markdown for analysis and review.
+
+This script is specifically designed for converting academic papers,
+organizing them, and preparing them for literature review workflows.
+"""
+
+import argparse
+import json
+import re
+import sys
+from datetime import datetime
+from pathlib import Path
+from typing import Dict, List
+
+from markitdown import MarkItDown
+
+
+def extract_metadata_from_filename(filename: str) -> Dict[str, str]:
+    """
+    Try to extract metadata from filename.
+    Supports patterns like: Author_Year_Title.pdf
+    """
+    metadata = {}
+
+    # Remove extension
+    name = Path(filename).stem
+
+    # Try to extract year
+    year_match = re.search(r'\b(19|20)\d{2}\b', name)
+    if year_match:
+        metadata['year'] = year_match.group()
+
+    # Split by underscores or dashes
+    parts = re.split(r'[_\-]', name)
+    if len(parts) >= 2:
+        metadata['author'] = parts[0].replace('_', ' ')
+        metadata['title'] = ' '.join(parts[1:]).replace('_', ' ')
+    else:
+        metadata['title'] = name.replace('_', ' ')
+
+    return metadata
+
+
+def convert_paper(
+    md: MarkItDown, input_file: Path, output_dir: Path, organize_by_year: bool = False
+) -> tuple[bool, Dict]:
+    """
+    Convert a single paper to Markdown with metadata extraction.
+
+    Args:
+        md: MarkItDown instance
+        input_file: Path to PDF file
+        output_dir: Output directory
+        organize_by_year: Organize into year subdirectories
+
+    Returns:
+        Tuple of (success, metadata_dict)
+    """
+    try:
+        print(f"Converting: {input_file.name}")
+
+        # Convert to Markdown
+        result = md.convert(str(input_file))
+
+        # Extract metadata from filename
+        metadata = extract_metadata_from_filename(input_file.name)
+        metadata['source_file'] = input_file.name
+        metadata['converted_date'] = datetime.now().isoformat()
+
+        # Try to extract title from content if not in filename
+        if 'title' not in metadata and result.title:
+            metadata['title'] = result.title
+
+        # Create output path
+        if organize_by_year and 'year' in metadata:
+            output_subdir = output_dir / metadata['year']
+            output_subdir.mkdir(parents=True, exist_ok=True)
+        else:
+            output_subdir = output_dir
+            output_subdir.mkdir(parents=True, exist_ok=True)
+
+        output_file = output_subdir / f"{input_file.stem}.md"
+
+        # Create formatted Markdown with front matter
+        content = "---\n"
+        content += f"title: \"{metadata.get('title', input_file.stem)}\"\n"
+        if 'author' in metadata:
+            content += f"author: \"{metadata['author']}\"\n"
+        if 'year' in metadata:
+            content += f"year: {metadata['year']}\n"
+        content += f"source: \"{metadata['source_file']}\"\n"
+        content += f"converted: \"{metadata['converted_date']}\"\n"
+        content += "---\n\n"
+
+        # Add title
+        content += f"# {metadata.get('title', input_file.stem)}\n\n"
+
+        # Add metadata section
+        content += "## Document Information\n\n"
+        if 'author' in metadata:
+            content += f"**Author**: {metadata['author']}\n"
+        if 'year' in metadata:
+            content += f"**Year**: {metadata['year']}\n"
+        content += f"**Source File**: {metadata['source_file']}\n"
+        content += f"**Converted**: {metadata['converted_date']}\n\n"
+        content += "---\n\n"
+
+        # Add content
+        content += result.text_content
+
+        # Write to file
+        output_file.write_text(content, encoding='utf-8')
+
+        print(f"✓ Saved to: {output_file}")
+
+        return True, metadata
+
+    except Exception as e:
+        print(f"✗ Error converting {input_file.name}: {str(e)}")
+        return False, {'source_file': input_file.name, 'error': str(e)}
+
+
+def create_index(papers: List[Dict], output_dir: Path):
+    """Create an index/catalog of all converted papers."""
+
+    # Sort by year (if available) and title
+    papers_sorted = sorted(papers, key=lambda x: (x.get('year', '9999'), x.get('title', '')))
+
+    # Create Markdown index
+    index_content = "# Literature Review Index\n\n"
+    index_content += f"**Generated**: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n"
+    index_content += f"**Total Papers**: {len(papers)}\n\n"
+    index_content += "---\n\n"
+
+    # Group by year
+    by_year = {}
+    for paper in papers_sorted:
+        year = paper.get('year', 'Unknown')
+        if year not in by_year:
+            by_year[year] = []
+        by_year[year].append(paper)
+
+    # Write by year
+    for year in sorted(by_year.keys()):
+        index_content += f"## {year}\n\n"
+        for paper in by_year[year]:
+            title = paper.get('title', paper.get('source_file', 'Unknown'))
+            author = paper.get('author', 'Unknown Author')
+            source = paper.get('source_file', '')
+
+            # Create link to markdown file
+            md_file = Path(source).stem + ".md"
+            if 'year' in paper and paper['year'] != 'Unknown':
+                md_file = f"{paper['year']}/{md_file}"
+
+            index_content += f"- **{title}**\n"
+            index_content += f"  - Author: {author}\n"
+            index_content += f"  - Source: {source}\n"
+            index_content += f"  - [Read Markdown]({md_file})\n\n"
+
+    # Write index
+    index_file = output_dir / "INDEX.md"
+    index_file.write_text(index_content, encoding='utf-8')
+    print(f"\n✓ Created index: {index_file}")
+
+    # Also create JSON catalog
+    catalog_file = output_dir / "catalog.json"
+    with open(catalog_file, 'w', encoding='utf-8') as f:
+        json.dump(papers_sorted, f, indent=2, ensure_ascii=False)
+    print(f"✓ Created catalog: {catalog_file}")
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Convert scientific literature PDFs to Markdown",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  # Convert all PDFs in a directory
+  python convert_literature.py papers/ output/
+  
+  # Organize by year
+  python convert_literature.py papers/ output/ --organize-by-year
+  
+  # Create index of all papers
+  python convert_literature.py papers/ output/ --create-index
+  
+Filename Conventions:
+  For best results, name your PDFs using this pattern:
+    Author_Year_Title.pdf
+    
+  Examples:
+    Smith_2023_Machine_Learning_Applications.pdf
+    Jones_2022_Climate_Change_Analysis.pdf
+        """,
+    )
+
+    parser.add_argument('input_dir', type=Path, help='Directory with PDF files')
+    parser.add_argument('output_dir', type=Path, help='Output directory for Markdown files')
+    parser.add_argument(
+        '--organize-by-year', '-y', action='store_true', help='Organize output into year subdirectories'
+    )
+    parser.add_argument('--create-index', '-i', action='store_true', help='Create an index/catalog of all papers')
+    parser.add_argument('--recursive', '-r', action='store_true', help='Search subdirectories recursively')
+
+    args = parser.parse_args()
+
+    # Validate input
+    if not args.input_dir.exists():
+        print(f"Error: Input directory '{args.input_dir}' does not exist")
+        sys.exit(1)
+
+    if not args.input_dir.is_dir():
+        print(f"Error: '{args.input_dir}' is not a directory")
+        sys.exit(1)
+
+    # Find PDF files
+    if args.recursive:
+        pdf_files = list(args.input_dir.rglob("*.pdf"))
+    else:
+        pdf_files = list(args.input_dir.glob("*.pdf"))
+
+    if not pdf_files:
+        print("No PDF files found")
+        sys.exit(1)
+
+    print(f"Found {len(pdf_files)} PDF file(s)")
+
+    # Create MarkItDown instance
+    md = MarkItDown()
+
+    # Convert all papers
+    results = []
+    success_count = 0
+
+    for pdf_file in pdf_files:
+        success, metadata = convert_paper(md, pdf_file, args.output_dir, args.organize_by_year)
+
+        if success:
+            success_count += 1
+            results.append(metadata)
+
+    # Create index if requested
+    if args.create_index and results:
+        create_index(results, args.output_dir)
+
+    # Print summary
+    print("\n" + "=" * 50)
+    print("CONVERSION SUMMARY")
+    print("=" * 50)
+    print(f"Total papers:    {len(pdf_files)}")
+    print(f"Successful:      {success_count}")
+    print(f"Failed:          {len(pdf_files) - success_count}")
+    print(f"Success rate:    {success_count / len(pdf_files) * 100:.1f}%")
+
+    sys.exit(0 if success_count == len(pdf_files) else 1)
+
+
+if __name__ == '__main__':
+    main()

package/bin/skills/markitdown/scripts/convert_with_ai.py

@@ -0,0 +1,224 @@
+#!/usr/bin/env python3
+"""
+Convert documents to Markdown with AI-enhanced image descriptions.
+
+This script demonstrates how to use MarkItDown with OpenRouter to generate
+detailed descriptions of images in documents (PowerPoint, PDFs with images, etc.)
+"""
+
+import argparse
+import os
+import sys
+from pathlib import Path
+
+from markitdown import MarkItDown
+from openai import OpenAI
+
+
+# Predefined prompts for different use cases
+PROMPTS = {
+    'scientific': """
+Analyze this scientific image or diagram. Provide:
+1. Type of visualization (graph, chart, microscopy, diagram, etc.)
+2. Key data points, trends, or patterns
+3. Axes labels, legends, and scales
+4. Notable features or findings
+5. Scientific context and significance
+Be precise, technical, and detailed.
+    """.strip(),
+    'presentation': """
+Describe this presentation slide image. Include:
+1. Main visual elements and their arrangement
+2. Key points or messages conveyed
+3. Data or information presented
+4. Visual hierarchy and emphasis
+Keep the description clear and informative.
+    """.strip(),
+    'general': """
+Describe this image in detail. Include:
+1. Main subjects and objects
+2. Visual composition and layout
+3. Text content (if any)
+4. Notable details
+5. Overall context and purpose
+Be comprehensive and accurate.
+    """.strip(),
+    'data_viz': """
+Analyze this data visualization. Provide:
+1. Type of chart/graph (bar, line, scatter, pie, etc.)
+2. Variables and axes
+3. Data ranges and scales
+4. Key patterns, trends, or outliers
+5. Statistical insights
+Focus on quantitative accuracy.
+    """.strip(),
+    'medical': """
+Describe this medical image. Include:
+1. Type of medical imaging (X-ray, MRI, CT, microscopy, etc.)
+2. Anatomical structures visible
+3. Notable findings or abnormalities
+4. Image quality and contrast
+5. Clinical relevance
+Be professional and precise.
+    """.strip(),
+}
+
+
+def convert_with_ai(
+    input_file: Path,
+    output_file: Path,
+    api_key: str,
+    model: str = "anthropic/claude-sonnet-4.5",
+    prompt_type: str = "general",
+    custom_prompt: str = None,
+) -> bool:
+    """
+    Convert a file to Markdown with AI image descriptions.
+
+    Args:
+        input_file: Path to input file
+        output_file: Path to output Markdown file
+        api_key: OpenRouter API key
+        model: Model name (default: anthropic/claude-sonnet-4.5)
+        prompt_type: Type of prompt to use
+        custom_prompt: Custom prompt (overrides prompt_type)
+
+    Returns:
+        True if successful, False otherwise
+    """
+    try:
+        # Initialize OpenRouter client (OpenAI-compatible)
+        client = OpenAI(api_key=api_key, base_url="https://openrouter.ai/api/v1")
+
+        # Select prompt
+        if custom_prompt:
+            prompt = custom_prompt
+        else:
+            prompt = PROMPTS.get(prompt_type, PROMPTS['general'])
+
+        print(f"Using model: {model}")
+        print(f"Prompt type: {prompt_type if not custom_prompt else 'custom'}")
+        print(f"Converting: {input_file}")
+
+        # Create MarkItDown with AI support
+        md = MarkItDown(llm_client=client, llm_model=model, llm_prompt=prompt)
+
+        # Convert file
+        result = md.convert(str(input_file))
+
+        # Create output with metadata
+        content = f"# {result.title or input_file.stem}\n\n"
+        content += f"**Source**: {input_file.name}\n"
+        content += f"**Format**: {input_file.suffix}\n"
+        content += f"**AI Model**: {model}\n"
+        content += f"**Prompt Type**: {prompt_type if not custom_prompt else 'custom'}\n\n"
+        content += "---\n\n"
+        content += result.text_content
+
+        # Write output
+        output_file.parent.mkdir(parents=True, exist_ok=True)
+        output_file.write_text(content, encoding='utf-8')
+
+        print(f"✓ Successfully converted to: {output_file}")
+        return True
+
+    except Exception as e:
+        print(f"✗ Error: {str(e)}", file=sys.stderr)
+        return False
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Convert documents to Markdown with AI-enhanced image descriptions",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Available prompt types:
+  scientific    - For scientific diagrams, graphs, and charts
+  presentation  - For presentation slides
+  general       - General-purpose image description
+  data_viz      - For data visualizations and charts
+  medical       - For medical imaging
+
+Examples:
+  # Convert a scientific paper
+  python convert_with_ai.py paper.pdf output.md --prompt-type scientific
+  
+  # Convert a presentation with custom model
+  python convert_with_ai.py slides.pptx slides.md --model anthropic/claude-sonnet-4.5 --prompt-type presentation
+  
+  # Use custom prompt with advanced vision model
+  python convert_with_ai.py diagram.png diagram.md --model anthropic/claude-sonnet-4.5 --custom-prompt "Describe this technical diagram"
+  
+  # Set API key via environment variable
+  export OPENROUTER_API_KEY="sk-or-v1-..."
+  python convert_with_ai.py image.jpg image.md
+
+Environment Variables:
+  OPENROUTER_API_KEY    OpenRouter API key (required if not passed via --api-key)
+
+Popular Models (use with --model):
+  anthropic/claude-sonnet-4.5 - Recommended for scientific vision
+  anthropic/claude-opus-4.5   - Advanced vision model
+  openai/gpt-4o              - GPT-4 Omni (vision support)
+  openai/gpt-4-vision        - GPT-4 Vision
+  google/gemini-pro-vision   - Gemini Pro Vision
+        """,
+    )
+
+    parser.add_argument('input', type=Path, help='Input file')
+    parser.add_argument('output', type=Path, help='Output Markdown file')
+    parser.add_argument('--api-key', '-k', help='OpenRouter API key (or set OPENROUTER_API_KEY env var)')
+    parser.add_argument(
+        '--model',
+        '-m',
+        default='anthropic/claude-sonnet-4.5',
+        help='Model to use via OpenRouter (default: anthropic/claude-sonnet-4.5)',
+    )
+    parser.add_argument(
+        '--prompt-type',
+        '-t',
+        choices=list(PROMPTS.keys()),
+        default='general',
+        help='Type of prompt to use (default: general)',
+    )
+    parser.add_argument('--custom-prompt', '-p', help='Custom prompt (overrides --prompt-type)')
+    parser.add_argument('--list-prompts', '-l', action='store_true', help='List available prompt types and exit')
+
+    args = parser.parse_args()
+
+    # List prompts and exit
+    if args.list_prompts:
+        print("Available prompt types:\n")
+        for name, prompt in PROMPTS.items():
+            print(f"[{name}]")
+            print(prompt)
+            print("\n" + "=" * 60 + "\n")
+        sys.exit(0)
+
+    # Get API key
+    api_key = args.api_key or os.environ.get('OPENROUTER_API_KEY')
+    if not api_key:
+        print("Error: OpenRouter API key required. Set OPENROUTER_API_KEY environment variable or use --api-key")
+        print("Get your API key at: https://openrouter.ai/keys")
+        sys.exit(1)
+
+    # Validate input file
+    if not args.input.exists():
+        print(f"Error: Input file '{args.input}' does not exist")
+        sys.exit(1)
+
+    # Convert file
+    success = convert_with_ai(
+        input_file=args.input,
+        output_file=args.output,
+        api_key=api_key,
+        model=args.model,
+        prompt_type=args.prompt_type,
+        custom_prompt=args.custom_prompt,
+    )
+
+    sys.exit(0 if success else 1)
+
+
+if __name__ == '__main__':
+    main()