lwazi 1.16.2 → 1.16.4

package/bin/crawl.php

@@ -96,8 +96,7 @@ if ($options['url'] === 'http://localhost' && $argc === 1) {
             $input = readline("Login email/username: ");
             $options['login_user'] = trim($input);
             
-            $input = readline("Login password: ");
-            $options['login_pass'] = trim($input);
+            $options['login_pass'] = readPasswordMasked("Login password: ");
         }
     }
     
@@ -197,6 +196,28 @@ function extractMeta(string $html, string $name): string {
     return '';
 }
 
+function extractText(string $html): string {
+    $dom = new DOMDocument();
+    libxml_use_internal_errors(true);
+    $dom->loadHTML($html);
+    libxml_clear_errors();
+
+    $remove = [];
+    foreach (['script', 'style', 'noscript'] as $tag) {
+        $nodes = $dom->getElementsByTagName($tag);
+        foreach ($nodes as $node) {
+            $remove[] = $node;
+        }
+    }
+    foreach ($remove as $node) {
+        $node->parentNode->removeChild($node);
+    }
+
+    $text = $dom->textContent ?? '';
+    $text = preg_replace('/\s+/', ' ', $text);
+    return trim($text);
+}
+
 function extractCsrfToken(string $html): ?string {
     if (preg_match('/name="_token"\s+value="([^"]+)"/', $html, $m)) return $m[1];
     if (preg_match('/name="_token"\s*value="([^"]+)"/', $html, $m)) return $m[1];
@@ -204,6 +225,163 @@ function extractCsrfToken(string $html): ?string {
     return null;
 }
 
+function isLoginPage(string $html): bool {
+    $dom = new DOMDocument();
+    libxml_use_internal_errors(true);
+    $dom->loadHTML($html);
+    libxml_clear_errors();
+
+    $inputs = $dom->getElementsByTagName('input');
+    foreach ($inputs as $input) {
+        $type = strtolower($input->getAttribute('type'));
+        if ($type === 'password') {
+            return true;
+        }
+    }
+    return false;
+}
+
+function fetchPage(string $url, string $cookies): array {
+    $headers = [];
+    $ch = curl_init();
+    curl_setopt($ch, CURLOPT_URL, $url);
+    curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
+    curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
+    curl_setopt($ch, CURLOPT_MAXREDIRS, 10);
+    curl_setopt($ch, CURLOPT_TIMEOUT, 30);
+    curl_setopt($ch, CURLOPT_USERAGENT, 'Lwazi/1.0');
+    curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
+    curl_setopt($ch, CURLOPT_COOKIEFILE, $cookies);
+    curl_setopt($ch, CURLOPT_HEADERFUNCTION, function ($ch, $headerLine) use (&$headers) {
+        $line = trim($headerLine);
+        if ($line !== '') {
+            $headers[] = $line;
+        }
+        return strlen($headerLine);
+    });
+
+    $html = curl_exec($ch);
+    $httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
+    $finalUrl = curl_getinfo($ch, CURLINFO_EFFECTIVE_URL) ?: $url;
+    $error = curl_error($ch);
+    curl_close($ch);
+
+    return [
+        'html' => $html ?: '',
+        'http_code' => $httpCode,
+        'final_url' => $finalUrl,
+        'headers' => $headers,
+        'error' => $error,
+    ];
+}
+
+function parseLoginForm(string $html, string $loginUrl): array {
+    $dom = new DOMDocument();
+    libxml_use_internal_errors(true);
+    $dom->loadHTML($html);
+    libxml_clear_errors();
+
+    $forms = $dom->getElementsByTagName('form');
+    $targetForm = null;
+    foreach ($forms as $form) {
+        $inputs = $form->getElementsByTagName('input');
+        foreach ($inputs as $input) {
+            if (strtolower($input->getAttribute('type')) === 'password') {
+                $targetForm = $form;
+                break 2;
+            }
+        }
+    }
+
+    $action = $loginUrl;
+    $method = 'post';
+    $fields = [];
+    $userField = null;
+    $passField = null;
+
+    if ($targetForm) {
+        $actionAttr = trim($targetForm->getAttribute('action'));
+        if ($actionAttr !== '') {
+            $action = normalizeUrl($actionAttr, $loginUrl);
+        }
+        $methodAttr = strtolower(trim($targetForm->getAttribute('method')));
+        if ($methodAttr !== '') {
+            $method = $methodAttr;
+        }
+
+        $inputs = $targetForm->getElementsByTagName('input');
+        foreach ($inputs as $input) {
+            $name = $input->getAttribute('name');
+            if ($name === '') {
+                continue;
+            }
+            $type = strtolower($input->getAttribute('type'));
+            $value = $input->getAttribute('value');
+            if ($type === 'hidden') {
+                $fields[$name] = $value;
+            }
+            if ($userField === null && in_array($type, ['text', 'email'], true)) {
+                if (preg_match('/email|user|username|login/i', $name)) {
+                    $userField = $name;
+                }
+            }
+            if ($type === 'password' && $passField === null) {
+                $passField = $name;
+            }
+        }
+    }
+
+    return [
+        'action' => $action,
+        'method' => $method,
+        'fields' => $fields,
+        'user_field' => $userField,
+        'pass_field' => $passField,
+    ];
+}
+
+function readPasswordMasked(string $prompt): string {
+    // Masked password input for POSIX terminals, with a safe fallback.
+    $sttyMode = null;
+    if (function_exists('shell_exec')) {
+        $sttyMode = shell_exec('stty -g 2>/dev/null');
+        if ($sttyMode) {
+            shell_exec('stty -echo -icanon min 1 time 0 2>/dev/null');
+        }
+    }
+
+    echo $prompt;
+    $password = '';
+
+    if ($sttyMode) {
+        while (true) {
+            $char = fread(STDIN, 1);
+            if ($char === false || $char === '') {
+                break;
+            }
+            if ($char === "\n" || $char === "\r") {
+                echo "\n";
+                break;
+            }
+            if ($char === "\x7f" || $char === "\x08") { // backspace/delete
+                if (strlen($password) > 0) {
+                    $password = substr($password, 0, -1);
+                    echo "\x08 \x08";
+                }
+                continue;
+            }
+            $password .= $char;
+            echo "*";
+        }
+        shell_exec('stty ' . $sttyMode . ' 2>/dev/null');
+        return $password;
+    }
+
+    // Fallback (no masking) if stty isn't available.
+    $line = fgets(STDIN);
+    return $line === false ? '' : rtrim($line, "\r\n");
+}
+
 function loginAndGetCookies(string $loginUrl, string $username, string $password, string $cookies): bool {
     echo "Attempting to login to $loginUrl...\n";
     
@@ -224,32 +402,28 @@ function loginAndGetCookies(string $loginUrl, string $username, string $password
         echo "Failed to fetch login page\n";
         return false;
     }
-    
+
+    $form = parseLoginForm($loginPage, $loginUrl);
     $csrfToken = extractCsrfToken($loginPage);
-    
-    $emailField = 'email';
-    $passwordField = 'password';
-    
-    if (preg_match('/name="(email|user|username|login)"[^>]*>/i', $loginPage, $m)) {
-        $emailField = $m[1];
-    }
-    if (preg_match('/name="(pass|pwd|password)"[^>]*>/i', $loginPage, $m)) {
-        $passwordField = $m[1];
-    }
-    
-    $loginData = [
-        $emailField => $username,
-        $passwordField => $password,
-    ];
-    
-    if ($csrfToken) {
+    $loginData = $form['fields'];
+
+    $userField = $form['user_field'] ?: 'email';
+    $passField = $form['pass_field'] ?: 'password';
+    $loginData[$userField] = $username;
+    $loginData[$passField] = $password;
+
+    if ($csrfToken && !isset($loginData['_token'])) {
         $loginData['_token'] = $csrfToken;
     }
     
     $ch = curl_init();
-    curl_setopt($ch, CURLOPT_URL, $loginUrl);
-    curl_setopt($ch, CURLOPT_POST, true);
-    curl_setopt($ch, CURLOPT_POSTFIELDS, http_build_query($loginData));
+    curl_setopt($ch, CURLOPT_URL, $form['action']);
+    curl_setopt($ch, CURLOPT_POST, ($form['method'] !== 'get'));
+    if ($form['method'] === 'get') {
+        curl_setopt($ch, CURLOPT_URL, $form['action'] . '?' . http_build_query($loginData));
+    } else {
+        curl_setopt($ch, CURLOPT_POSTFIELDS, http_build_query($loginData));
+    }
     curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
     curl_setopt($ch, CURLOPT_COOKIEJAR, $cookies);
     curl_setopt($ch, CURLOPT_COOKIEFILE, $cookies);
@@ -262,15 +436,28 @@ function loginAndGetCookies(string $loginUrl, string $username, string $password
     $httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
     curl_close($ch);
     
-    if ($httpCode >= 200 && $httpCode < 400) {
+    if ($httpCode >= 200 && $httpCode < 400 && $result && !isLoginPage($result)) {
         echo "Login successful!\n";
         return true;
     }
-    
+
     echo "Login may have failed (HTTP $httpCode)\n";
     return false;
 }
 
+function ensureSessionAlive(string $checkUrl, string $loginUrl, string $loginUser, string $loginPass, string $cookies): bool {
+    if (!$loginUrl || !$loginUser || !$loginPass) {
+        return false;
+    }
+
+    $probe = fetchPage($checkUrl, $cookies);
+    if ($probe['html'] && !isLoginPage($probe['html'])) {
+        return true;
+    }
+
+    return loginAndGetCookies($loginUrl, $loginUser, $loginPass, $cookies);
+}
+
 if ($loginUrl && $loginUser && $loginPass) {
     $loggedIn = loginAndGetCookies($loginUrl, $loginUser, $loginPass, $cookies);
     if (!$loggedIn) {
@@ -289,6 +476,8 @@ $adjacency = [];
 $pageData = [];
 
 $totalPages = 0;
+$keepAliveInterval = 120;
+$lastKeepAliveAt = time();
 
 while (!empty($queue) && $totalPages < $maxPages) {
     [$url, $depth] = array_shift($queue);
@@ -302,33 +491,48 @@ while (!empty($queue) && $totalPages < $maxPages) {
     $totalPages++;
     
     echo "Crawling [$totalPages/$maxPages]: $url (depth: $depth)\n";
-    
-    $ch = curl_init();
-    curl_setopt($ch, CURLOPT_URL, $url);
-    curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
-    curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
-    curl_setopt($ch, CURLOPT_MAXREDIRS, 10);
-    curl_setopt($ch, CURLOPT_TIMEOUT, 30);
-    curl_setopt($ch, CURLOPT_USERAGENT, 'Lwazi/1.0');
-    curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
-    curl_setopt($ch, CURLOPT_COOKIEFILE, $cookies);
-    
-    $html = curl_exec($ch);
-    $httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
-    curl_close($ch);
+
+    if ($loginUrl && $loginUser && $loginPass) {
+        $now = time();
+        if (($now - $lastKeepAliveAt) >= $keepAliveInterval) {
+            ensureSessionAlive($rootUrl, $loginUrl, $loginUser, $loginPass, $cookies);
+            $lastKeepAliveAt = $now;
+        }
+    }
+
+    $response = fetchPage($url, $cookies);
+    $html = $response['html'];
+    $httpCode = $response['http_code'];
+    $finalUrl = $response['final_url'];
+    $headers = $response['headers'];
     
     if (!$html || $httpCode >= 400) {
         echo "  -> Failed (HTTP $httpCode)\n";
         continue;
     }
     
-    if (stripos($html, 'login') !== false && stripos($html, 'password') !== false) {
-        echo "  -> Login required, skipping\n";
-        continue;
+    if ($html && isLoginPage($html)) {
+        if ($loginUrl && $loginUser && $loginPass) {
+            echo "  -> Login required, re-authenticating and retrying\n";
+            $loggedIn = loginAndGetCookies($loginUrl, $loginUser, $loginPass, $cookies);
+            if ($loggedIn) {
+                $response = fetchPage($url, $cookies);
+                $html = $response['html'];
+                $httpCode = $response['http_code'];
+                $finalUrl = $response['final_url'];
+                $headers = $response['headers'];
+            }
+        }
+
+        if (!$html || isLoginPage($html)) {
+            echo "  -> Login required, skipping\n";
+            continue;
+        }
     }
     
     $title = extractTitle($html);
     $description = extractMeta($html, 'description');
+    $text = extractText($html);
     $links = extractLinks($html, $url);
     
     $nodes[$urlId] = [
@@ -342,6 +546,13 @@ while (!empty($queue) && $totalPages < $maxPages) {
     $pageData[$urlId] = [
         'html' => $html,
         'url' => $url,
+        'final_url' => $finalUrl,
+        'title' => $title,
+        'description' => $description,
+        'text' => $text,
+        'headers' => $headers,
+        'fetched_at' => date('c'),
+        'http_code' => $httpCode,
     ];
     
     foreach ($links as $link) {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "lwazi",
-    "version": "1.16.2",
+    "version": "1.16.4",
     "description": "Lwazi is an AI assistant for Laravel. Install with one command to add an AI assistant to your Laravel app.",
     "main": "bin/lwazi.js",
     "bin": {

package/uninstall

@@ -62,11 +62,18 @@ if [ -f "$INDEX_FILE" ]; then
         php -r '
         $file = "public/index.php";
         $content = file_get_contents($file);
+        $hadNewline = str_ends_with($content, "\n");
         $content = preg_replace(
             "/\n*require __DIR__\.\"\/..\/lwazi\/auto_prepend\.php\";\n*/",
-            "\n\n",
+            "\n",
             $content
         );
+        $content = preg_replace("/\n{3,}/", "\n\n", $content);
+        if ($hadNewline) {
+            $content = rtrim($content, "\n") . "\n";
+        } else {
+            $content = rtrim($content, "\n");
+        }
         file_put_contents($file, $content);
         '
         echo "Removed injection from public/index.php"
@@ -79,11 +86,18 @@ if [ -f "$PROJECT_DIR/index.php" ]; then
         php -r '
         $file = "index.php";
         $content = file_get_contents($file);
+        $hadNewline = str_ends_with($content, "\n");
         $content = preg_replace(
             "/\n*require __DIR__\.\"\/lwazi\/auto_prepend\.php\";\n*/",
-            "\n\n",
+            "\n",
             $content
         );
+        $content = preg_replace("/\n{3,}/", "\n\n", $content);
+        if ($hadNewline) {
+            $content = rtrim($content, "\n") . "\n";
+        } else {
+            $content = rtrim($content, "\n");
+        }
         file_put_contents($file, $content);
         '
         echo "Removed injection from index.php"
@@ -104,6 +118,7 @@ php -r '
 $file = "composer.json";
 if (file_exists($file)) {
     $json = json_decode(file_get_contents($file), true);
+    $hadNewline = str_ends_with(file_get_contents($file), "\n");
     $changed = false;
     
     if (isset($json["require"]["lwazi/core"])) {
@@ -130,7 +145,11 @@ if (file_exists($file)) {
     }
     
     if ($changed) {
-        file_put_contents($file, json_encode($json, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES));
+        $encoded = json_encode($json, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES);
+        if ($hadNewline) {
+            $encoded .= "\n";
+        }
+        file_put_contents($file, $encoded);
         echo "Removed lwazi from composer.json\n";
     }
 }

package/uninstall.bat

@@ -22,6 +22,15 @@ echo.
 echo Uninstalling Lwazi...
 echo -------------------------------------------
 
+REM Prefer PowerShell uninstaller if available for cleaner cleanup
+where powershell >nul 2>&1
+if %ERRORLEVEL%==0 (
+    if exist "%PROJECT_DIR%\uninstall.ps1" (
+        powershell -NoProfile -ExecutionPolicy Bypass -File "%PROJECT_DIR%\uninstall.ps1" -Force
+        exit /b %ERRORLEVEL%
+    )
+)
+
 REM Remove lwazi folder
 if exist "%PROJECT_DIR%\lwazi" (
     rmdir /s /q "%PROJECT_DIR%\lwazi"

package/uninstall.ps1

@@ -7,6 +7,30 @@ param(
 
 $ErrorActionPreference = "Stop"
 
+function Get-LineEnding {
+    param([string]$Content)
+    if ($Content -match "`r`n") { return "`r`n" }
+    return "`n"
+}
+
+function Write-FilePreserveNewline {
+    param(
+        [string]$Path,
+        [string]$Content,
+        [string]$OriginalContent
+    )
+    $lineEnding = Get-LineEnding -Content $OriginalContent
+    $hadNewline = $OriginalContent -match "(`r?`n)$"
+
+    if ($hadNewline) {
+        $Content = $Content.TrimEnd("`r", "`n") + $lineEnding
+    } else {
+        $Content = $Content.TrimEnd("`r", "`n")
+    }
+
+    [IO.File]::WriteAllText($Path, $Content)
+}
+
 Write-Host "==========================================" -ForegroundColor Cyan
 Write-Host "  Lwazi AI Assistant Uninstaller" -ForegroundColor Cyan
 Write-Host "==========================================" -ForegroundColor Cyan
@@ -28,16 +52,6 @@ Write-Host ""
 Write-Host "Uninstalling Lwazi..." -ForegroundColor Yellow
 Write-Host "-------------------------------------------"
 
-# Check if composer was used
-$composerUsed = $false
-$modeFile = "$ProjectDir\lwazi\.lwazi_mode"
-if (Test-Path $modeFile) {
-    $modeContent = Get-Content $modeFile -Raw
-    if ($modeContent -match "mode=true") {
-        $composerUsed = $true
-    }
-}
-
 # Remove lwazi folder
 if (Test-Path "$ProjectDir\lwazi") {
     Remove-Item -Recurse -Force "$ProjectDir\lwazi"
@@ -50,7 +64,8 @@ if (Test-Path "$ProjectDir\.env") {
     if ($envContent -match "LWAZI_") {
         $lines = Get-Content "$ProjectDir\.env"
         $newLines = $lines | Where-Object { $_ -notmatch "^LWAZI_" -and $_ -notmatch "^# Lwazi AI Configuration" }
-        Set-Content "$ProjectDir\.env" -Value $newLines
+        $content = $newLines -join (Get-LineEnding -Content $envContent)
+        Write-FilePreserveNewline -Path "$ProjectDir\.env" -Content $content -OriginalContent $envContent
         Write-Host "Removed Lwazi config from .env" -ForegroundColor Green
     }
 }
@@ -62,7 +77,8 @@ if (Test-Path $userIni) {
     if ($userIniContent -match "auto_prepend_file.*lwazi") {
         $lines = Get-Content $userIni
         $newLines = $lines | Where-Object { $_ -notmatch "auto_prepend_file.*lwazi" }
-        Set-Content $userIni -Value $newLines
+        $content = $newLines -join (Get-LineEnding -Content $userIniContent)
+        Write-FilePreserveNewline -Path $userIni -Content $content -OriginalContent $userIniContent
         Write-Host "Removed auto_prepend_file from .user.ini" -ForegroundColor Green
     }
 }
@@ -72,8 +88,9 @@ $indexFile = "$ProjectDir\public\index.php"
 if (Test-Path $indexFile) {
     $content = Get-Content $indexFile -Raw
     if ($content -match "lwazi/auto_prepend.php") {
+        $original = $content
         $content = $content -replace "`r?`n?require __DIR__`.\"/`../lwazi/auto_prepend.php`";`r?`n?", "`r`n"
-        Set-Content -Path $indexFile -Value $content -NoNewline
+        Write-FilePreserveNewline -Path $indexFile -Content $content -OriginalContent $original
         Write-Host "Removed injection from public\index.php" -ForegroundColor Green
     }
 }
@@ -83,8 +100,9 @@ $rootIndexFile = "$ProjectDir\index.php"
 if (Test-Path $rootIndexFile) {
     $content = Get-Content $rootIndexFile -Raw
     if ($content -match "lwazi/auto_prepend.php") {
+        $original = $content
         $content = $content -replace "`r?`n?require __DIR__`.\"/lwazi/auto_prepend.php`";`r?`n?", "`r`n"
-        Set-Content -Path $rootIndexFile -Value $content -NoNewline
+        Write-FilePreserveNewline -Path $rootIndexFile -Content $content -OriginalContent $original
         Write-Host "Removed injection from index.php" -ForegroundColor Green
     }
 }
@@ -96,46 +114,46 @@ if (Test-Path "$ProjectDir\artisan") {
     Write-Host "Cleared caches" -ForegroundColor Green
 }
 
-# Remove composer integration if it was used
-if ($composerUsed) {
-    Write-Host "Cleaning up composer integration..." -ForegroundColor Yellow
-    
-    $composerJsonFile = "$ProjectDir\composer.json"
-    if (Test-Path $composerJsonFile) {
-        $composerJson = Get-Content $composerJsonFile | ConvertFrom-Json
-        $changed = $false
-        
-        if ($composerJson.PSObject.Properties.Name -contains "require") {
-            if ($composerJson.require.PSObject.Properties.Name -contains "lwazi/core") {
-                $composerJson.require.PSObject.Properties.Remove("lwazi/core")
-                $changed = $true
-                Write-Host "Removed lwazi/core from composer.json" -ForegroundColor Green
-            }
+# Always clean up composer.json (lwazi might have added it even if mode wasn't saved)
+Write-Host "Cleaning up composer.json..." -ForegroundColor Yellow
+
+$composerJsonFile = "$ProjectDir\composer.json"
+if (Test-Path $composerJsonFile) {
+    $composerJson = Get-Content $composerJsonFile | ConvertFrom-Json
+    $changed = $false
+
+    if ($composerJson.PSObject.Properties.Name -contains "require") {
+        if ($composerJson.require.PSObject.Properties.Name -contains "lwazi/core") {
+            $composerJson.require.PSObject.Properties.Remove("lwazi/core")
+            $changed = $true
+            Write-Host "Removed lwazi/core from composer.json" -ForegroundColor Green
         }
-        
-        if ($composerJson.PSObject.Properties.Name -contains "repositories" -and $composerJson.repositories) {
-            $originalCount = $composerJson.repositories.Count
-            $composerJson.repositories = $composerJson.repositories | Where-Object { $_.url -ne "lwazi" }
-            if ($composerJson.repositories.Count -ne $originalCount) {
-                $changed = $true
-            }
-            if ($composerJson.repositories.Count -eq 0) {
-                $composerJson.PSObject.Properties.Remove("repositories")
-            }
+    }
+
+    if ($composerJson.PSObject.Properties.Name -contains "repositories" -and $composerJson.repositories) {
+        $originalCount = $composerJson.repositories.Count
+        $composerJson.repositories = $composerJson.repositories | Where-Object { $_.url -ne "lwazi" }
+        if ($composerJson.repositories.Count -ne $originalCount) {
+            $changed = $true
         }
-        
-        if ($changed) {
-            $composerJson | ConvertTo-Json -Depth 10 | Set-Content $composerJsonFile -Encoding UTF8
+        if ($composerJson.repositories.Count -eq 0) {
+            $composerJson.PSObject.Properties.Remove("repositories")
         }
     }
-    
-    # Run composer dump-autoload
-    $composerCmd = Get-Command composer -ErrorAction SilentlyContinue
-    if ($composerCmd) {
-        composer dump-autoload 2>&1 | Out-Null
+
+    if ($changed) {
+        $original = Get-Content $composerJsonFile -Raw
+        $jsonText = $composerJson | ConvertTo-Json -Depth 10
+        Write-FilePreserveNewline -Path $composerJsonFile -Content $jsonText -OriginalContent $original
     }
 }
 
+# Run composer dump-autoload
+$composerCmd = Get-Command composer -ErrorAction SilentlyContinue
+if ($composerCmd) {
+    composer dump-autoload 2>&1 | Out-Null
+}
+
 Write-Host ""
 Write-Host "==========================================" -ForegroundColor Cyan
 Write-Host "  Uninstallation Complete!" -ForegroundColor Cyan