arkaos 2.15.0 → 2.16.0

package/departments/kb/skills/knowledge/scripts/kb-cleanup.sh

@@ -0,0 +1,135 @@
+#!/bin/bash
+# ============================================================================
+# ARKA OS — KB Media Cleanup
+# Removes old media directories for completed jobs.
+#
+# Usage: kb-cleanup.sh [--older-than 90d] [--dry-run]
+# ============================================================================
+
+ARKA_DIR="$HOME/.arka-os"
+MEDIA_DIR="$ARKA_DIR/media"
+JOBS_FILE="$ARKA_DIR/kb-jobs.json"
+LOCK_FILE="$ARKA_DIR/kb-jobs.lock"
+
+MAX_DAYS=90
+DRY_RUN=false
+
+# Parse args
+while [ $# -gt 0 ]; do
+    case "$1" in
+        --older-than)
+            MAX_DAYS=$(echo "${2:-90d}" | sed 's/d$//')
+            shift 2
+            ;;
+        --dry-run)
+            DRY_RUN=true
+            shift
+            ;;
+        *)
+            shift
+            ;;
+    esac
+done
+
+CYAN='\033[0;36m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+RED='\033[0;31m'
+NC='\033[0m'
+
+if [ ! -d "$MEDIA_DIR" ]; then
+    echo -e "${YELLOW}No media directory found.${NC}"
+    exit 0
+fi
+
+if ! command -v jq &>/dev/null; then
+    echo "Error: jq not found. Install with: brew install jq" >&2
+    exit 1
+fi
+
+echo ""
+echo -e "${CYAN}═══ ARKA OS — KB Media Cleanup ═══${NC}"
+echo -e "  Removing completed jobs older than ${CYAN}${MAX_DAYS} days${NC}"
+if $DRY_RUN; then
+    echo -e "  ${YELLOW}DRY RUN — no files will be deleted${NC}"
+fi
+echo ""
+
+CUTOFF_DATE=$(date -v-${MAX_DAYS}d +%Y-%m-%d 2>/dev/null || date -d "-${MAX_DAYS} days" +%Y-%m-%d 2>/dev/null)
+if [ -z "$CUTOFF_DATE" ]; then
+    echo "Error: Could not calculate cutoff date" >&2
+    exit 1
+fi
+
+echo -e "  Cutoff date: $CUTOFF_DATE"
+echo ""
+
+TOTAL_FREED=0
+DIRS_REMOVED=0
+
+# Iterate date directories
+for date_dir in "$MEDIA_DIR"/*/; do
+    [ -d "$date_dir" ] || continue
+    DIR_DATE=$(basename "$date_dir")
+
+    # Check if date directory is older than cutoff
+    if [[ "$DIR_DATE" > "$CUTOFF_DATE" ]] || [[ "$DIR_DATE" == "$CUTOFF_DATE" ]]; then
+        continue
+    fi
+
+    # Check each job directory inside the date dir
+    for job_dir in "$date_dir"*/; do
+        [ -d "$job_dir" ] || continue
+        JOB_ID=$(basename "$job_dir")
+
+        # Check if this job is completed in the jobs file
+        if [ -f "$JOBS_FILE" ]; then
+            STATUS=$(jq -r --arg id "$JOB_ID" '.jobs[] | select(.id == $id) | .status // "unknown"' "$JOBS_FILE" 2>/dev/null)
+
+            # Only remove completed jobs
+            if [ "$STATUS" != "completed" ] && [ "$STATUS" != "unknown" ]; then
+                echo -e "  ${YELLOW}SKIP${NC} $DIR_DATE/$JOB_ID (status: $STATUS)"
+                continue
+            fi
+        fi
+
+        # Calculate directory size
+        DIR_SIZE=$(du -sk "$job_dir" 2>/dev/null | cut -f1)
+        DIR_SIZE_MB=$(echo "scale=1; $DIR_SIZE / 1024" | bc 2>/dev/null || echo "$DIR_SIZE KB")
+
+        if $DRY_RUN; then
+            echo -e "  ${YELLOW}WOULD REMOVE${NC} $DIR_DATE/$JOB_ID (${DIR_SIZE_MB}MB)"
+        else
+            rm -rf "$job_dir"
+            echo -e "  ${GREEN}REMOVED${NC} $DIR_DATE/$JOB_ID (${DIR_SIZE_MB}MB)"
+
+            # Remove job from jobs file
+            if [ -f "$JOBS_FILE" ]; then
+                (
+                    flock -x 200
+                    tmp="$JOBS_FILE.tmp.$$"
+                    jq --arg id "$JOB_ID" '.jobs = [.jobs[] | select(.id != $id)]' \
+                        "$JOBS_FILE" > "$tmp" && mv "$tmp" "$JOBS_FILE"
+                ) 200>"$LOCK_FILE"
+            fi
+        fi
+
+        TOTAL_FREED=$((TOTAL_FREED + DIR_SIZE))
+        DIRS_REMOVED=$((DIRS_REMOVED + 1))
+    done
+
+    # Remove empty date directories
+    if ! $DRY_RUN; then
+        rmdir "$date_dir" 2>/dev/null || true
+    fi
+done
+
+# Summary
+echo ""
+FREED_MB=$(echo "scale=1; $TOTAL_FREED / 1024" | bc 2>/dev/null || echo "${TOTAL_FREED}KB")
+if $DRY_RUN; then
+    echo -e "${YELLOW}DRY RUN:${NC} Would remove ${CYAN}$DIRS_REMOVED${NC} directories, freeing ~${CYAN}${FREED_MB}MB${NC}"
+else
+    echo -e "${GREEN}Cleaned up:${NC} ${CYAN}$DIRS_REMOVED${NC} directories, freed ~${CYAN}${FREED_MB}MB${NC}"
+fi
+echo -e "${CYAN}══════════════════════════════════${NC}"

package/departments/kb/skills/knowledge/scripts/kb-queue.sh

@@ -0,0 +1,156 @@
+#!/bin/bash
+# ============================================================================
+# ARKA OS — KB Queue Dispatcher
+# Queues a URL for background download + transcription.
+#
+# Usage: kb-queue.sh <url> [--persona "Name"]
+# Output: job ID (8 chars)
+# ============================================================================
+set -euo pipefail
+
+URL="${1:-}"
+PERSONA=""
+
+# Parse args
+shift || true
+while [ $# -gt 0 ]; do
+    case "$1" in
+        --persona) PERSONA="${2:-}"; shift 2 ;;
+        *) shift ;;
+    esac
+done
+
+if [ -z "$URL" ]; then
+    echo "Usage: kb-queue.sh <url> [--persona \"Name\"]" >&2
+    exit 1
+fi
+
+ARKA_DIR="$HOME/.arka-os"
+JOBS_FILE="$ARKA_DIR/kb-jobs.json"
+LOCK_FILE="$ARKA_DIR/kb-jobs.lock"
+CAPS_FILE="$ARKA_DIR/capabilities.json"
+SCRIPTS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+mkdir -p "$ARKA_DIR"
+
+# ─── Generate job ID ────────────────────────────────────────────────────────
+
+JOB_ID=$(printf '%s%s' "$URL" "$(date +%s%N)" | md5sum 2>/dev/null || printf '%s%s' "$URL" "$(date +%s)" | md5 2>/dev/null)
+JOB_ID="${JOB_ID:0:8}"
+
+# ─── Create output directory ────────────────────────────────────────────────
+
+DATE_DIR=$(date +%Y-%m-%d)
+OUTPUT_DIR="$ARKA_DIR/media/$DATE_DIR/$JOB_ID"
+mkdir -p "$OUTPUT_DIR"
+
+# ─── Determine transcription method ─────────────────────────────────────────
+
+TRANSCRIPTION_METHOD="none"
+
+# Run capabilities check if file doesn't exist or is older than 1 hour
+REFRESH_CAPS=false
+if [ ! -f "$CAPS_FILE" ]; then
+    REFRESH_CAPS=true
+elif [ "$(uname)" = "Darwin" ]; then
+    CAPS_AGE=$(( $(date +%s) - $(stat -f%m "$CAPS_FILE") ))
+    [ "$CAPS_AGE" -gt 3600 ] && REFRESH_CAPS=true
+else
+    CAPS_AGE=$(( $(date +%s) - $(stat -c%Y "$CAPS_FILE") ))
+    [ "$CAPS_AGE" -gt 3600 ] && REFRESH_CAPS=true
+fi
+
+if $REFRESH_CAPS && [ -f "$SCRIPTS_DIR/kb-check-capabilities.sh" ]; then
+    bash "$SCRIPTS_DIR/kb-check-capabilities.sh" >/dev/null 2>&1
+fi
+
+# Read transcription method from capabilities
+if [ -f "$CAPS_FILE" ] && command -v jq &>/dev/null; then
+    TRANSCRIPTION_METHOD=$(jq -r '.transcription.method // "none"' "$CAPS_FILE")
+fi
+
+# ─── Check yt-dlp is available ──────────────────────────────────────────────
+
+if ! command -v yt-dlp &>/dev/null; then
+    echo "Error: yt-dlp not found. Install with: brew install yt-dlp" >&2
+    exit 1
+fi
+
+# ─── Initialize jobs file if needed ─────────────────────────────────────────
+
+(
+    flock -x 200
+
+    if [ ! -f "$JOBS_FILE" ]; then
+        echo '{"jobs":[]}' > "$JOBS_FILE"
+    fi
+
+    # Validate JSON
+    if ! jq empty "$JOBS_FILE" 2>/dev/null; then
+        echo '{"jobs":[]}' > "$JOBS_FILE"
+    fi
+
+    # Add job entry
+    NOW=$(date -u +%Y-%m-%dT%H:%M:%SZ)
+    tmp="$JOBS_FILE.tmp.$$"
+    jq --arg id "$JOB_ID" \
+       --arg url "$URL" \
+       --arg dir "$OUTPUT_DIR" \
+       --arg method "$TRANSCRIPTION_METHOD" \
+       --arg persona "$PERSONA" \
+       --arg now "$NOW" \
+       '.jobs += [{
+            "id": $id,
+            "url": $url,
+            "output_dir": $dir,
+            "transcription_method": $method,
+            "persona": $persona,
+            "status": "queued",
+            "title": null,
+            "word_count": null,
+            "error": null,
+            "pid": null,
+            "created_at": $now,
+            "updated_at": $now
+        }]' "$JOBS_FILE" > "$tmp" && mv "$tmp" "$JOBS_FILE"
+
+) 200>"$LOCK_FILE"
+
+# ─── Launch worker in background ────────────────────────────────────────────
+
+WORKER_SCRIPT="$SCRIPTS_DIR/kb-worker.sh"
+if [ ! -f "$WORKER_SCRIPT" ]; then
+    echo "Error: Worker script not found: $WORKER_SCRIPT" >&2
+    exit 1
+fi
+
+nohup bash "$WORKER_SCRIPT" "$JOB_ID" "$URL" "$OUTPUT_DIR" "$TRANSCRIPTION_METHOD" \
+    > /dev/null 2>&1 &
+WORKER_PID=$!
+
+# Record PID in jobs file
+(
+    flock -x 200
+    tmp="$JOBS_FILE.tmp.$$"
+    jq --arg id "$JOB_ID" --argjson pid "$WORKER_PID" \
+        '(.jobs[] | select(.id == $id)).pid = $pid' \
+        "$JOBS_FILE" > "$tmp" && mv "$tmp" "$JOBS_FILE"
+) 200>"$LOCK_FILE"
+
+# ─── Output ─────────────────────────────────────────────────────────────────
+
+if [ -t 1 ]; then
+    CYAN='\033[0;36m'
+    GREEN='\033[0;32m'
+    NC='\033[0m'
+    echo -e "${GREEN}✓${NC} Queued job ${CYAN}$JOB_ID${NC}"
+    echo -e "  URL:    $URL"
+    echo -e "  Method: $TRANSCRIPTION_METHOD"
+    echo -e "  Output: $OUTPUT_DIR"
+    echo -e "  PID:    $WORKER_PID"
+    if [ -n "$PERSONA" ]; then
+        echo -e "  Persona: $PERSONA"
+    fi
+else
+    echo "$JOB_ID"
+fi

package/departments/kb/skills/knowledge/scripts/kb-status.sh

@@ -0,0 +1,195 @@
+#!/bin/bash
+# ============================================================================
+# ARKA OS — KB Status Checker
+# Shows status of queued/running/ready/completed jobs.
+#
+# Usage: kb-status.sh [job-id] [--json]
+# ============================================================================
+
+ARKA_DIR="$HOME/.arka-os"
+JOBS_FILE="$ARKA_DIR/kb-jobs.json"
+LOCK_FILE="$ARKA_DIR/kb-jobs.lock"
+
+JOB_ID=""
+JSON_MODE=false
+
+# Parse args
+while [ $# -gt 0 ]; do
+    case "$1" in
+        --json) JSON_MODE=true; shift ;;
+        *)      JOB_ID="$1"; shift ;;
+    esac
+done
+
+# Colors
+CYAN='\033[0;36m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+RED='\033[0;31m'
+BLUE='\033[0;34m'
+NC='\033[0m'
+
+# ─── Check jobs file exists ─────────────────────────────────────────────────
+
+if [ ! -f "$JOBS_FILE" ]; then
+    if $JSON_MODE; then
+        echo '{"jobs":[]}'
+    else
+        echo -e "${YELLOW}No jobs found.${NC} Queue a URL with: /kb learn <url>"
+    fi
+    exit 0
+fi
+
+if ! command -v jq &>/dev/null; then
+    echo "Error: jq not found. Install with: brew install jq" >&2
+    exit 1
+fi
+
+# ─── Validate running PIDs ──────────────────────────────────────────────────
+
+(
+    flock -x 200
+
+    # Check each running/downloading/transcribing job's PID
+    ACTIVE_IDS=$(jq -r '.jobs[] | select(.status == "downloading" or .status == "transcribing" or .status == "queued") | .id' "$JOBS_FILE" 2>/dev/null)
+
+    CHANGED=false
+    for id in $ACTIVE_IDS; do
+        PID=$(jq -r --arg id "$id" '.jobs[] | select(.id == $id) | .pid // empty' "$JOBS_FILE")
+        if [ -n "$PID" ] && ! kill -0 "$PID" 2>/dev/null; then
+            # PID is dead — mark as failed
+            NOW=$(date -u +%Y-%m-%dT%H:%M:%SZ)
+            tmp="$JOBS_FILE.tmp.$$"
+            jq --arg id "$id" --arg t "$NOW" \
+                '(.jobs[] | select(.id == $id)) |= (.status = "failed" | .error = "Worker process died unexpectedly" | .updated_at = $t)' \
+                "$JOBS_FILE" > "$tmp" && mv "$tmp" "$JOBS_FILE"
+            CHANGED=true
+        fi
+    done
+
+) 200>"$LOCK_FILE"
+
+# ─── Single job detail ──────────────────────────────────────────────────────
+
+if [ -n "$JOB_ID" ]; then
+    JOB=$(jq --arg id "$JOB_ID" '.jobs[] | select(.id == $id)' "$JOBS_FILE" 2>/dev/null)
+
+    if [ -z "$JOB" ] || [ "$JOB" = "null" ]; then
+        echo "Job not found: $JOB_ID" >&2
+        exit 1
+    fi
+
+    if $JSON_MODE; then
+        echo "$JOB" | jq .
+        exit 0
+    fi
+
+    STATUS=$(echo "$JOB" | jq -r '.status')
+    URL=$(echo "$JOB" | jq -r '.url')
+    TITLE=$(echo "$JOB" | jq -r '.title // "Pending..."')
+    DIR=$(echo "$JOB" | jq -r '.output_dir')
+    METHOD=$(echo "$JOB" | jq -r '.transcription_method')
+    PERSONA=$(echo "$JOB" | jq -r '.persona // "Not assigned"')
+    WORDS=$(echo "$JOB" | jq -r '.word_count // "—"')
+    ERROR=$(echo "$JOB" | jq -r '.error // ""')
+    CREATED=$(echo "$JOB" | jq -r '.created_at')
+    UPDATED=$(echo "$JOB" | jq -r '.updated_at')
+
+    # Status color
+    case "$STATUS" in
+        queued)       STATUS_COLOR="$YELLOW" ;;
+        downloading)  STATUS_COLOR="$BLUE" ;;
+        transcribing) STATUS_COLOR="$BLUE" ;;
+        ready)        STATUS_COLOR="$GREEN" ;;
+        analyzing)    STATUS_COLOR="$CYAN" ;;
+        completed)    STATUS_COLOR="$GREEN" ;;
+        failed)       STATUS_COLOR="$RED" ;;
+        *)            STATUS_COLOR="$NC" ;;
+    esac
+
+    echo ""
+    echo -e "${CYAN}═══ Job: $JOB_ID ═══${NC}"
+    echo -e "  Status:    ${STATUS_COLOR}$STATUS${NC}"
+    echo -e "  Title:     $TITLE"
+    echo -e "  URL:       $URL"
+    echo -e "  Method:    $METHOD"
+    echo -e "  Persona:   $PERSONA"
+    echo -e "  Words:     $WORDS"
+    echo -e "  Output:    $DIR"
+    echo -e "  Created:   $CREATED"
+    echo -e "  Updated:   $UPDATED"
+    if [ -n "$ERROR" ]; then
+        echo -e "  ${RED}Error:     $ERROR${NC}"
+    fi
+
+    # Show available files
+    if [ -d "$DIR" ]; then
+        echo ""
+        echo -e "  ${CYAN}Files:${NC}"
+        for f in "$DIR"/*; do
+            [ -f "$f" ] || continue
+            SIZE=$(ls -lh "$f" | awk '{print $5}')
+            echo -e "    $(basename "$f") ($SIZE)"
+        done
+    fi
+
+    echo -e "${CYAN}══════════════════════${NC}"
+    exit 0
+fi
+
+# ─── Summary table ──────────────────────────────────────────────────────────
+
+if $JSON_MODE; then
+    jq . "$JOBS_FILE"
+    exit 0
+fi
+
+TOTAL=$(jq '.jobs | length' "$JOBS_FILE")
+if [ "$TOTAL" -eq 0 ]; then
+    echo -e "${YELLOW}No jobs found.${NC} Queue a URL with: /kb learn <url>"
+    exit 0
+fi
+
+echo ""
+echo -e "${CYAN}═══ ARKA OS — KB Job Queue ═══${NC}"
+echo ""
+printf "  ${CYAN}%-10s %-12s %-40s %s${NC}\n" "ID" "STATUS" "TITLE" "METHOD"
+printf "  %-10s %-12s %-40s %s\n" "──────────" "────────────" "────────────────────────────────────────" "──────────"
+
+jq -r '.jobs[] | [.id, .status, (.title // "Pending..."), .transcription_method] | @tsv' "$JOBS_FILE" | while IFS=$'\t' read -r id status title method; do
+    # Truncate title
+    if [ ${#title} -gt 38 ]; then
+        title="${title:0:35}..."
+    fi
+
+    # Status color
+    case "$status" in
+        queued)       SC="$YELLOW" ;;
+        downloading)  SC="$BLUE" ;;
+        transcribing) SC="$BLUE" ;;
+        ready)        SC="$GREEN" ;;
+        analyzing)    SC="$CYAN" ;;
+        completed)    SC="$GREEN" ;;
+        failed)       SC="$RED" ;;
+        *)            SC="$NC" ;;
+    esac
+
+    printf "  %-10s ${SC}%-12s${NC} %-40s %s\n" "$id" "$status" "$title" "$method"
+done
+
+# Summary counts
+QUEUED=$(jq '[.jobs[] | select(.status == "queued")] | length' "$JOBS_FILE")
+ACTIVE=$(jq '[.jobs[] | select(.status == "downloading" or .status == "transcribing")] | length' "$JOBS_FILE")
+READY=$(jq '[.jobs[] | select(.status == "ready")] | length' "$JOBS_FILE")
+COMPLETED=$(jq '[.jobs[] | select(.status == "completed")] | length' "$JOBS_FILE")
+FAILED=$(jq '[.jobs[] | select(.status == "failed")] | length' "$JOBS_FILE")
+
+echo ""
+echo -e "  Queued: ${YELLOW}$QUEUED${NC}  Active: ${BLUE}$ACTIVE${NC}  Ready: ${GREEN}$READY${NC}  Completed: ${GREEN}$COMPLETED${NC}  Failed: ${RED}$FAILED${NC}"
+echo ""
+
+if [ "$READY" -gt 0 ]; then
+    echo -e "  ${GREEN}$READY job(s) ready for processing.${NC} Run: /kb process <job-id>"
+fi
+
+echo -e "${CYAN}═══════════════════════════════${NC}"