crowe-logic 0.1.0

package/requirements.txt

@@ -0,0 +1,11 @@
+azure-ai-agents>=1.1.0
+azure-identity>=1.17.0
+azure-ai-projects>=2.0.0b4
+opentelemetry-sdk
+azure-core-tracing-opentelemetry
+click>=8.1.0
+rich>=13.0.0
+prompt-toolkit>=3.0.0
+python-dotenv>=1.0.0
+httpx>=0.27.0
+beautifulsoup4>=4.12.0

package/scripts/create_agent.py

@@ -0,0 +1,163 @@
+#!/usr/bin/env python3
+"""
+Crowe Logic Agent — Create & Configure
+
+Creates the Crowe Logic agent on Azure AI Foundry with all available tools:
+- Custom function tools (filesystem, shell, browser, search)
+- Code Interpreter (sandboxed Python execution)
+- Bing Grounding (live web search via Azure)
+- File Search (RAG over uploaded documents)
+- Azure AI Search (vector search over knowledge base)
+
+Usage:
+    python scripts/create_agent.py
+    python scripts/create_agent.py --name "crowe-logic-v2" --verbose
+"""
+
+import os
+import sys
+import json
+import argparse
+
+# Add project root to path
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from azure.ai.agents import AgentsClient
+from azure.ai.agents.models import (
+    CodeInterpreterTool,
+    FileSearchTool,
+    BingGroundingTool,
+    AzureAISearchTool,
+    AzureAISearchQueryType,
+    FunctionTool,
+    ToolSet,
+)
+from azure.identity import DefaultAzureCredential
+
+from config.agent_config import (
+    PROJECT_ENDPOINT,
+    MODEL_DEPLOYMENT_NAME,
+    SYSTEM_INSTRUCTIONS,
+    AGENT_NAME,
+    AGENT_VERSION,
+    BING_CONNECTION_ID,
+    AI_SEARCH_CONNECTION_ID,
+    AI_SEARCH_INDEX_NAME,
+)
+from tools import user_functions
+
+
+def create_agent(name: str = AGENT_NAME, verbose: bool = False):
+    """Create the Crowe Logic agent with all available tools."""
+
+    print(f"  Connecting to Azure AI Foundry at {PROJECT_ENDPOINT}")
+    client = AgentsClient(
+        endpoint=PROJECT_ENDPOINT,
+        credential=DefaultAzureCredential(),
+    )
+
+    # Build the toolset with all available tools
+    toolset = ToolSet()
+
+    # 1. Custom function tools (Claude-like local capabilities)
+    functions = FunctionTool(user_functions)
+    toolset.add(functions)
+    print("  Added custom function tools (filesystem, shell, browser, search)")
+
+    # 2. Code Interpreter (sandboxed Python)
+    code_interpreter = CodeInterpreterTool()
+    toolset.add(code_interpreter)
+    print("  Added Code Interpreter")
+
+    # Enable auto-execution of function calls
+    client.enable_auto_function_calls(toolset)
+
+    # Build additional tools list (for tools that use tools+tool_resources pattern)
+    extra_tools = []
+    extra_resources = {}
+
+    # 3. Bing Grounding (web search via Azure connection)
+    if BING_CONNECTION_ID:
+        bing = BingGroundingTool(connection_id=BING_CONNECTION_ID)
+        extra_tools.extend(bing.definitions)
+        print(f"  Added Bing Grounding (connection: {BING_CONNECTION_ID[:20]}...)")
+    else:
+        print("  Skipped Bing Grounding (no AZURE_BING_CONNECTION_ID set)")
+
+    # 4. Azure AI Search (vector search over knowledge base)
+    if AI_SEARCH_CONNECTION_ID:
+        ai_search = AzureAISearchTool(
+            index_connection_id=AI_SEARCH_CONNECTION_ID,
+            index_name=AI_SEARCH_INDEX_NAME,
+            query_type=AzureAISearchQueryType.SEMANTIC,
+            top_k=5,
+        )
+        extra_tools.extend(ai_search.definitions)
+        if hasattr(ai_search, "resources") and ai_search.resources:
+            extra_resources.update(ai_search.resources)
+        print(f"  Added Azure AI Search (index: {AI_SEARCH_INDEX_NAME})")
+    else:
+        print("  Skipped Azure AI Search (no AI_AZURE_AI_CONNECTION_ID set)")
+
+    # Create the agent
+    print(f"\n  Creating agent '{name}' with model '{MODEL_DEPLOYMENT_NAME}'...")
+
+    create_kwargs = dict(
+        model=MODEL_DEPLOYMENT_NAME,
+        name=name,
+        instructions=SYSTEM_INSTRUCTIONS,
+        toolset=toolset,
+    )
+
+    # Add extra tools if any connection-based tools were configured
+    if extra_tools:
+        create_kwargs["tools"] = extra_tools
+    if extra_resources:
+        create_kwargs["tool_resources"] = extra_resources
+
+    agent = client.create_agent(**create_kwargs)
+
+    print(f"\n  Agent created successfully!")
+    print(f"  Agent ID:   {agent.id}")
+    print(f"  Agent Name: {agent.name}")
+    print(f"  Model:      {agent.model}")
+
+    # Save agent ID for CLI use
+    agent_file = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), ".agent_id")
+    with open(agent_file, "w") as f:
+        json.dump({"agent_id": agent.id, "name": name, "version": AGENT_VERSION, "model": MODEL_DEPLOYMENT_NAME}, f, indent=2)
+    print(f"  Saved agent ID to .agent_id")
+
+    if verbose:
+        print(f"\n  Full agent config:")
+        print(f"  Instructions: {SYSTEM_INSTRUCTIONS[:200]}...")
+        print(f"  Tools: {[t.__class__.__name__ for t in [functions, code_interpreter]]}")
+
+    return agent
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Create the Crowe Logic agent on Azure AI Foundry")
+    parser.add_argument("--name", default=AGENT_NAME, help="Agent name")
+    parser.add_argument("--verbose", "-v", action="store_true", help="Verbose output")
+    args = parser.parse_args()
+
+    print(f"\n{'='*60}")
+    print(f"  CROWE LOGIC AGENT — CREATE")
+    print(f"  Version {AGENT_VERSION}")
+    print(f"{'='*60}\n")
+
+    try:
+        agent = create_agent(name=args.name, verbose=args.verbose)
+        print(f"\n{'='*60}")
+        print(f"  READY — Run: crowe-logic chat")
+        print(f"{'='*60}\n")
+    except Exception as e:
+        print(f"\n  ERROR: {e}")
+        print(f"  Make sure you've run: az login")
+        print(f"  And set PROJECT_ENDPOINT in .env")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

package/scripts/fine_tune.py

@@ -0,0 +1,271 @@
+#!/usr/bin/env python3
+"""
+Crowe Logic Agent — Fine-Tuning Pipeline
+
+Converts CroweLM unified datasets to Azure AI Foundry format and
+initiates fine-tuning of gpt-oss-120b with domain knowledge:
+  - Biotech / Pharma / Drug Discovery
+  - Mycology / Mushroom Cultivation
+  - Molecular Biology / Gene/RNA/Protein
+  - Scientific Coding / Reasoning
+
+Usage:
+    python scripts/fine_tune.py convert     # Convert datasets to Azure format
+    python scripts/fine_tune.py upload       # Upload to Azure AI Foundry
+    python scripts/fine_tune.py train        # Start fine-tuning job
+    python scripts/fine_tune.py status       # Check training status
+    python scripts/fine_tune.py pipeline     # Run full pipeline
+"""
+
+import os
+import sys
+import json
+import argparse
+from pathlib import Path
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from config.agent_config import PROJECT_ENDPOINT, MODEL_DEPLOYMENT_NAME
+
+# Dataset paths
+DATA_DIR = Path(__file__).parent.parent / "data"
+CROWELM_UNIFIED = DATA_DIR / "crowelm-unified"
+CROWELM_BIOTECH = DATA_DIR / "crowelm-biotech"
+OUTPUT_DIR = DATA_DIR / "azure-ft"
+
+
+def convert_nemo_sft_to_openai(input_path: Path, output_path: Path, max_samples: int = 0):
+    """
+    Convert NeMo SFT JSONL to OpenAI chat completion format for Azure fine-tuning.
+
+    NeMo SFT format:
+        {"conversations": [{"role": "user", "content": "..."}, {"role": "assistant", "content": "..."}]}
+
+    Azure/OpenAI format:
+        {"messages": [{"role": "system", "content": "..."}, {"role": "user", "content": "..."}, {"role": "assistant", "content": "..."}]}
+    """
+    print(f"  Converting: {input_path.name}")
+
+    system_msg = {
+        "role": "system",
+        "content": (
+            "You are CroweLM, an expert AI assistant specializing in biotech, "
+            "pharmaceutical science, mycology, mushroom cultivation, molecular biology, "
+            "drug discovery, and scientific research. You provide accurate, detailed, "
+            "and actionable information grounded in scientific literature and practical experience. "
+            "Created by Crowe Logic, Inc."
+        )
+    }
+
+    converted = 0
+    skipped = 0
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+
+    with open(input_path, "r", encoding="utf-8") as fin, \
+         open(output_path, "w", encoding="utf-8") as fout:
+        for line in fin:
+            if max_samples and converted >= max_samples:
+                break
+            try:
+                entry = json.loads(line.strip())
+
+                # Handle different input formats
+                messages = [system_msg]
+
+                if "conversations" in entry:
+                    # NeMo SFT format
+                    for turn in entry["conversations"]:
+                        role = turn.get("role", turn.get("from", "user"))
+                        content = turn.get("content", turn.get("value", ""))
+                        if role in ("human", "user"):
+                            messages.append({"role": "user", "content": content})
+                        elif role in ("gpt", "assistant", "model"):
+                            messages.append({"role": "assistant", "content": content})
+
+                elif "instruction" in entry and "output" in entry:
+                    # Instruction format
+                    user_content = entry["instruction"]
+                    if entry.get("input"):
+                        user_content += f"\n\n{entry['input']}"
+                    messages.append({"role": "user", "content": user_content})
+                    messages.append({"role": "assistant", "content": entry["output"]})
+
+                elif "question" in entry and "answer" in entry:
+                    # QA format
+                    messages.append({"role": "user", "content": entry["question"]})
+                    messages.append({"role": "assistant", "content": entry["answer"]})
+
+                elif "text" in entry:
+                    # Pretraining text — wrap as a knowledge entry
+                    messages.append({"role": "user", "content": "Explain the following topic in detail."})
+                    messages.append({"role": "assistant", "content": entry["text"]})
+
+                else:
+                    skipped += 1
+                    continue
+
+                # Validate: must have at least system + user + assistant
+                if len(messages) >= 3:
+                    fout.write(json.dumps({"messages": messages}, ensure_ascii=False) + "\n")
+                    converted += 1
+                else:
+                    skipped += 1
+
+            except (json.JSONDecodeError, KeyError):
+                skipped += 1
+
+    print(f"    Converted: {converted:,} | Skipped: {skipped:,}")
+    return converted
+
+
+def cmd_convert(args):
+    """Convert all CroweLM datasets to Azure fine-tuning format."""
+    print(f"\n{'='*60}")
+    print(f"  DATASET CONVERSION — CroweLM → Azure FT Format")
+    print(f"{'='*60}\n")
+
+    total = 0
+
+    # Find all JSONL files in the unified dataset
+    jsonl_files = list(CROWELM_UNIFIED.rglob("*.jsonl"))
+    if not jsonl_files:
+        print("  No JSONL files found in data/crowelm-unified/")
+        print("  Looking in Azure blob storage...")
+        # Try downloading from Azure
+        print("  Run: az storage blob download-batch --account-name crowelmdata7595 --source nvidia-curated-datasets -d data/crowelm-unified/")
+        return
+
+    for jsonl_file in sorted(jsonl_files):
+        output_name = f"azure_ft_{jsonl_file.stem}.jsonl"
+        output_path = OUTPUT_DIR / output_name
+        count = convert_nemo_sft_to_openai(
+            jsonl_file, output_path,
+            max_samples=args.max_samples if hasattr(args, 'max_samples') else 0
+        )
+        total += count
+
+    print(f"\n  Total converted: {total:,} samples")
+    print(f"  Output directory: {OUTPUT_DIR}")
+
+    # Create a merged training file
+    merged_path = OUTPUT_DIR / "crowelm_merged_train.jsonl"
+    print(f"\n  Merging all files into: {merged_path.name}")
+    with open(merged_path, "w") as fout:
+        for ft_file in sorted(OUTPUT_DIR.glob("azure_ft_*.jsonl")):
+            with open(ft_file) as fin:
+                for line in fin:
+                    fout.write(line)
+
+    line_count = sum(1 for _ in open(merged_path))
+    print(f"  Merged file: {line_count:,} samples")
+
+
+def cmd_upload(args):
+    """Upload converted dataset to Azure AI Foundry."""
+    print(f"\n{'='*60}")
+    print(f"  DATASET UPLOAD — Azure AI Foundry")
+    print(f"{'='*60}\n")
+
+    from azure.ai.agents import AgentsClient
+    from azure.ai.agents.models import FilePurpose
+    from azure.identity import DefaultAzureCredential
+
+    merged_path = OUTPUT_DIR / "crowelm_merged_train.jsonl"
+    if not merged_path.exists():
+        print("  ERROR: Run 'convert' first to generate the training file.")
+        return
+
+    client = AgentsClient(
+        endpoint=PROJECT_ENDPOINT,
+        credential=DefaultAzureCredential(),
+    )
+
+    print(f"  Uploading: {merged_path.name}")
+    file = client.files.upload_and_poll(
+        file_path=str(merged_path),
+        purpose=FilePurpose.AGENTS,
+    )
+    print(f"  Uploaded! File ID: {file.id}")
+
+    # Save file ID for training
+    meta_path = OUTPUT_DIR / "upload_meta.json"
+    with open(meta_path, "w") as f:
+        json.dump({"file_id": file.id, "filename": merged_path.name}, f, indent=2)
+    print(f"  Saved metadata to: {meta_path}")
+
+
+def cmd_train(args):
+    """Start fine-tuning job on Azure (placeholder — requires Azure ML or OpenAI FT API)."""
+    print(f"\n{'='*60}")
+    print(f"  FINE-TUNING — gpt-oss-120b + CroweLM Data")
+    print(f"{'='*60}\n")
+
+    print("  NOTE: gpt-oss-120b fine-tuning options:")
+    print()
+    print("  1. Azure AI Foundry Managed Fine-Tuning:")
+    print("     az ml job create --file training_config.yaml")
+    print()
+    print("  2. Self-hosted via vLLM + LoRA (RunPod / local H100):")
+    print("     See data/crowelm-unified/RUNPOD_TRAINING_GUIDE.md")
+    print()
+    print("  3. HuggingFace Transformers + PEFT:")
+    print("     python scripts/hf_fine_tune.py")
+    print()
+
+    merged_path = OUTPUT_DIR / "crowelm_merged_train.jsonl"
+    if merged_path.exists():
+        line_count = sum(1 for _ in open(merged_path))
+        size_mb = merged_path.stat().st_size / (1024 * 1024)
+        print(f"  Dataset ready: {line_count:,} samples ({size_mb:.1f} MB)")
+    else:
+        print("  Dataset not converted yet. Run: python scripts/fine_tune.py convert")
+
+
+def cmd_status(args):
+    """Check fine-tuning job status."""
+    meta_path = OUTPUT_DIR / "upload_meta.json"
+    if meta_path.exists():
+        with open(meta_path) as f:
+            meta = json.load(f)
+        print(f"  Uploaded file: {meta.get('file_id', 'unknown')}")
+    else:
+        print("  No upload metadata found.")
+
+
+def cmd_pipeline(args):
+    """Run full pipeline: convert → upload."""
+    cmd_convert(args)
+    cmd_upload(args)
+    cmd_train(args)
+
+
+def main():
+    parser = argparse.ArgumentParser(description="CroweLM Fine-Tuning Pipeline")
+    sub = parser.add_subparsers(dest="command")
+
+    p_convert = sub.add_parser("convert", help="Convert datasets to Azure format")
+    p_convert.add_argument("--max-samples", type=int, default=0, help="Limit samples (0=all)")
+
+    sub.add_parser("upload", help="Upload to Azure AI Foundry")
+    sub.add_parser("train", help="Start fine-tuning")
+    sub.add_parser("status", help="Check status")
+    sub.add_parser("pipeline", help="Full pipeline")
+
+    args = parser.parse_args()
+
+    if not args.command:
+        parser.print_help()
+        return
+
+    commands = {
+        "convert": cmd_convert,
+        "upload": cmd_upload,
+        "train": cmd_train,
+        "status": cmd_status,
+        "pipeline": cmd_pipeline,
+    }
+    commands[args.command](args)
+
+
+if __name__ == "__main__":
+    main()

package/scripts/npm-publish.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+# Quick npm publish with live OTP input
+cd ~/Projects/crowe-logic-foundry
+echo ""
+echo "=== CROWE LOGIC — npm Publish ==="
+echo ""
+read -p "Enter your authenticator OTP code: " otp
+npm publish --access public --otp="$otp"
+if [ $? -eq 0 ]; then
+    echo ""
+    echo "Published! https://www.npmjs.com/package/crowe-logic"
+else
+    echo ""
+    echo "Failed — try again with: bash scripts/npm-publish.sh"
+fi

package/scripts/orchestrator.applescript

@@ -0,0 +1,162 @@
+-- ============================================================
+-- Crowe Logic Agent — macOS AppleScript Orchestrator
+-- ============================================================
+-- One-click automation for agent development workflow:
+--   1. Deploy/update the agent on Azure AI Foundry
+--   2. Launch interactive chat in Terminal
+--   3. Open Azure dashboard for monitoring
+--   4. Run test suite
+--   5. Full pipeline (deploy + test + chat)
+--
+-- Usage:
+--   osascript scripts/orchestrator.applescript deploy
+--   osascript scripts/orchestrator.applescript chat
+--   osascript scripts/orchestrator.applescript dashboard
+--   osascript scripts/orchestrator.applescript test
+--   osascript scripts/orchestrator.applescript pipeline
+--   osascript scripts/orchestrator.applescript (shows menu)
+-- ============================================================
+
+property projectPath : "/Users/crowelogic/Projects/crowe-logic-foundry"
+property venvPath : "/Users/crowelogic/Projects/crowe-logic-foundry/.venv"
+property azureDashboard : "https://ai.azure.com/nextgen/r/mVYda6I5Q7uWtGK2g4Co8A,rg-crowelogicos-7858,,crowelogicos-7858-resource,crowelogicos-7858/build/agents/crowe-logic/build"
+property pythonCmd : "source .venv/bin/activate 2>/dev/null || true && python3"
+
+-- ============================================================
+-- MAIN ENTRY POINT
+-- ============================================================
+
+on run argv
+	if (count of argv) > 0 then
+		set action to item 1 of argv
+	else
+		set action to showMenu()
+	end if
+
+	if action is "deploy" then
+		doDeployAgent()
+	else if action is "chat" then
+		doLaunchChat()
+	else if action is "dashboard" then
+		doOpenDashboard()
+	else if action is "test" then
+		doRunTests()
+	else if action is "pipeline" then
+		doFullPipeline()
+	else if action is "setup" then
+		doInitialSetup()
+	else if action is "status" then
+		doShowStatus()
+	else
+		display dialog "Unknown action: " & action buttons {"OK"} default button "OK" with icon caution
+	end if
+end run
+
+-- ============================================================
+-- MENU
+-- ============================================================
+
+on showMenu()
+	set menuItems to {"Deploy Agent", "Launch Chat", "Open Azure Dashboard", "Run Tests", "Full Pipeline (Deploy + Test + Chat)", "Initial Setup", "Show Status"}
+	set menuActions to {"deploy", "chat", "dashboard", "test", "pipeline", "setup", "status"}
+
+	set chosen to choose from list menuItems with prompt "Crowe Logic Agent — Choose Action:" with title "Crowe Logic" default items {"Launch Chat"}
+
+	if chosen is false then
+		error number -128 -- User cancelled
+	end if
+
+	set chosenItem to item 1 of chosen
+	repeat with i from 1 to count of menuItems
+		if item i of menuItems is chosenItem then
+			return item i of menuActions
+		end if
+	end repeat
+
+	return "chat"
+end showMenu
+
+-- ============================================================
+-- ACTIONS
+-- ============================================================
+
+on doInitialSetup()
+	display notification "Setting up Crowe Logic..." with title "Crowe Logic"
+
+	runInTerminal("cd " & quoted form of projectPath & " && " & ¬
+		"python3 -m venv .venv && " & ¬
+		"source .venv/bin/activate && " & ¬
+		"pip install -r requirements.txt && " & ¬
+		"pip install -e . && " & ¬
+		"echo '' && echo '========================================' && " & ¬
+		"echo '  Setup complete!' && " & ¬
+		"echo '  Next: cp .env.example .env && edit .env' && " & ¬
+		"echo '  Then: crowe-logic deploy' && " & ¬
+		"echo '========================================'")
+
+	display notification "Setup complete! Edit .env next." with title "Crowe Logic"
+end doInitialSetup
+
+on doDeployAgent()
+	display notification "Deploying Crowe Logic agent..." with title "Crowe Logic"
+
+	runInTerminal("cd " & quoted form of projectPath & " && " & ¬
+		pythonCmd & " scripts/create_agent.py --verbose")
+
+	display notification "Agent deployed successfully!" with title "Crowe Logic" sound name "Glass"
+end doDeployAgent
+
+on doLaunchChat()
+	runInTerminal("cd " & quoted form of projectPath & " && " & ¬
+		pythonCmd & " -m cli.crowe_logic chat")
+end doLaunchChat
+
+on doOpenDashboard()
+	open location azureDashboard
+	display notification "Azure AI Foundry dashboard opened" with title "Crowe Logic"
+end doOpenDashboard
+
+on doRunTests()
+	display notification "Running agent tests..." with title "Crowe Logic"
+
+	runInTerminal("cd " & quoted form of projectPath & " && " & ¬
+		pythonCmd & " scripts/test_agent.py")
+end doRunTests
+
+on doFullPipeline()
+	display notification "Starting full pipeline..." with title "Crowe Logic"
+
+	-- Step 1: Deploy
+	runInTerminal("cd " & quoted form of projectPath & " && " & ¬
+		pythonCmd & " scripts/create_agent.py && " & ¬
+		"echo '' && echo 'Agent deployed. Running tests...' && " & ¬
+		"echo '' && " & ¬
+		pythonCmd & " scripts/test_agent.py && " & ¬
+		"echo '' && echo '========================================' && " & ¬
+		"echo '  Pipeline complete! Starting chat...' && " & ¬
+		"echo '========================================' && " & ¬
+		"echo '' && " & ¬
+		pythonCmd & " -m cli.crowe_logic chat")
+
+	-- Step 2: Open dashboard in background
+	delay 2
+	open location azureDashboard
+
+	display notification "Pipeline complete!" with title "Crowe Logic" sound name "Glass"
+end doFullPipeline
+
+on doShowStatus()
+	runInTerminal("cd " & quoted form of projectPath & " && " & ¬
+		pythonCmd & " -m cli.crowe_logic status")
+end doShowStatus
+
+-- ============================================================
+-- HELPERS
+-- ============================================================
+
+on runInTerminal(cmd)
+	tell application "Terminal"
+		activate
+		do script cmd
+	end tell
+end runInTerminal