@crewx/cli 0.9.0-rc.139 → 0.9.0-rc.140

package/dist/commands/db.d.ts

@@ -7,5 +7,6 @@
  *   crewx db push --dry-run      Show preview only, no changes
  */
 export declare const TASK_LOG_MIGRATION_HELP = "Usage:\n  crewx db migrate-task-logs --dry-run [--db PATH]\n  crewx db migrate-task-logs --apply [--db PATH] [--batch-tasks N]\n  crewx db migrate-task-logs --verify [--db PATH]\n\nModes:\n  --dry-run  Read and validate legacy blobs without writing or creating a backup.\n  --apply    Create a consistent SQLite backup, then migrate one task per transaction.\n  --verify   Check source invariants, event sequences, counts, and orphan rows.\n\nApply policy:\n  Only blob tasks with status success, failed, or completed are migrated.\n  pending, running, paused, and unknown statuses are deferred for a later run.\n  There is no --force or --include-active option. Apply requires the estimated\n  database/event growth plus a 10 GiB free-space reserve.\n\nExit codes:\n  0  Completed with no malformed rows or invariant failures.\n  1  Safe tasks completed but a row/invariant failed, or apply was blocked.\n  2  Invalid command-line arguments.\n\nThe command never runs automatically at server startup and never runs VACUUM.";
+export declare const REQUEST_LOG_MIGRATION_HELP = "Usage:\n  crewx db migrate-request-logs [--batch N] [--dry-run]\n\nMoves the request log rows still stored in crewx.db into logs.db, the file next\nto it. Each distinct request/response body is stored once in logs.db, keyed by\nits SHA-256; a row points to its bodies by hash, so nothing is dropped.\n\nOptions:\n  --batch N   Rows per batch (default 250; a batch is also capped at about 8 MB).\n  --dry-run   Count rows and distinct bodies without writing anything (logs.db is not created).\n\nSafe while crewx is running: each batch is committed to logs.db, verified there,\nand only then deleted from crewx.db in one short transaction. Interrupt it and run\nit again at any time. The database is chosen like everywhere else (CREWX_DB, else\n~/.crewx/crewx.db). No VACUUM and no DROP: crewx.db keeps its size until a later compaction.\n\nExit codes:\n  0  Every row is in logs.db and none is left in crewx.db (or the dry run finished).\n  1  Rows are still in crewx.db, or an error stopped the move (progress is kept).\n  2  Invalid command-line arguments.";
 export declare const USAGE_BACKFILL_HELP = "Usage:\n  crewx db backfill-usage [--dry-run|--apply|--verify] [--db PATH] [--backup PATH]\n  crewx db backfill-usage --restore SNAPSHOT [--db PATH]\n\nModes:\n  --dry-run  Show provider/reason recovery plans without changing the database (default).\n  --apply    Create a row-level undo snapshot, then account one task per transaction.\n  --verify   Recheck remaining targets and usageAccounting markers without changing the database.\n  --restore  Restore rows from SNAPSHOT when their current values still match the snapshot after-state.\n\nApply policy:\n  Only success, failed, and completed tasks with zero recorded usage are considered.\n  Workflow/MCP anchors and plugin/mock rows are excluded. Rows completed within 24 hours\n  are not marked unavailable when their logs do not contain usage.\n\nExit codes:\n  0  Completed successfully.\n  1  Apply was blocked or one or more task updates failed.\n  2  Invalid command-line arguments.";
 export declare function handleDb(args: string[]): Promise<void>;

package/dist/commands/db.js

@@ -11,7 +11,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.USAGE_BACKFILL_HELP = exports.TASK_LOG_MIGRATION_HELP = void 0;
+exports.USAGE_BACKFILL_HELP = exports.REQUEST_LOG_MIGRATION_HELP = exports.TASK_LOG_MIGRATION_HELP = void 0;
 exports.handleDb = handleDb;
 const path_1 = __importDefault(require("path"));
 const os_1 = __importDefault(require("os"));
@@ -20,6 +20,7 @@ const repository_1 = require("@crewx/sdk/repository");
 const repository_2 = require("@crewx/sdk/repository");
 const repository_3 = require("@crewx/sdk/repository");
 const repository_4 = require("@crewx/sdk/repository");
+const repository_5 = require("@crewx/sdk/repository");
 exports.TASK_LOG_MIGRATION_HELP = `Usage:
   crewx db migrate-task-logs --dry-run [--db PATH]
   crewx db migrate-task-logs --apply [--db PATH] [--batch-tasks N]
@@ -42,6 +43,26 @@ Exit codes:
   2  Invalid command-line arguments.
 
 The command never runs automatically at server startup and never runs VACUUM.`;
+exports.REQUEST_LOG_MIGRATION_HELP = `Usage:
+  crewx db migrate-request-logs [--batch N] [--dry-run]
+
+Moves the request log rows still stored in crewx.db into logs.db, the file next
+to it. Each distinct request/response body is stored once in logs.db, keyed by
+its SHA-256; a row points to its bodies by hash, so nothing is dropped.
+
+Options:
+  --batch N   Rows per batch (default 250; a batch is also capped at about 8 MB).
+  --dry-run   Count rows and distinct bodies without writing anything (logs.db is not created).
+
+Safe while crewx is running: each batch is committed to logs.db, verified there,
+and only then deleted from crewx.db in one short transaction. Interrupt it and run
+it again at any time. The database is chosen like everywhere else (CREWX_DB, else
+~/.crewx/crewx.db). No VACUUM and no DROP: crewx.db keeps its size until a later compaction.
+
+Exit codes:
+  0  Every row is in logs.db and none is left in crewx.db (or the dry run finished).
+  1  Rows are still in crewx.db, or an error stopped the move (progress is kept).
+  2  Invalid command-line arguments.`;
 exports.USAGE_BACKFILL_HELP = `Usage:
   crewx db backfill-usage [--dry-run|--apply|--verify] [--db PATH] [--backup PATH]
   crewx db backfill-usage --restore SNAPSHOT [--db PATH]
@@ -122,6 +143,10 @@ async function handleDb(args) {
         await handleTaskLogMigration(args.slice(1));
         return;
     }
+    if (subcommand === 'migrate-request-logs') {
+        await handleRequestLogMigration(args.slice(1));
+        return;
+    }
     if (subcommand === 'backfill-usage') {
         await handleUsageBackfill(args.slice(1));
         return;
@@ -129,6 +154,7 @@ async function handleDb(args) {
     if (subcommand === '--help' || subcommand === '-h') {
         console.log('Usage: crewx db push [--force] [--dry-run]');
         console.log(exports.TASK_LOG_MIGRATION_HELP);
+        console.log(exports.REQUEST_LOG_MIGRATION_HELP);
         return;
     }
     if (!subcommand || subcommand === 'push') {
@@ -292,6 +318,132 @@ async function handleTaskLogMigration(args) {
         process.exitCode = 1;
     }
 }
+class RequestLogMigrationUsageError extends Error {
+}
+function parseRequestLogMigrationArgs(args) {
+    let batchRows;
+    let dryRun = false;
+    const parseBatch = (value) => {
+        const parsed = Number(value);
+        if (!value || !Number.isSafeInteger(parsed) || parsed < 1) {
+            throw new RequestLogMigrationUsageError('--batch must be a positive integer.');
+        }
+        return parsed;
+    };
+    for (let index = 0; index < args.length; index += 1) {
+        const arg = args[index];
+        if (arg === '--help' || arg === '-h')
+            throw new RequestLogMigrationUsageError(exports.REQUEST_LOG_MIGRATION_HELP);
+        if (arg === '--dry-run') {
+            dryRun = true;
+            continue;
+        }
+        if (arg === '--batch') {
+            batchRows = parseBatch(args[index + 1]);
+            index += 1;
+            continue;
+        }
+        if (arg.startsWith('--batch=')) {
+            batchRows = parseBatch(arg.slice('--batch='.length));
+            continue;
+        }
+        throw new RequestLogMigrationUsageError(`Unknown option: ${arg}`);
+    }
+    return { batchRows, dryRun };
+}
+function formatBytes(bytes) {
+    if (bytes < 1024)
+        return `${bytes} B`;
+    const units = ['KiB', 'MiB', 'GiB', 'TiB'];
+    let value = bytes;
+    let unit = -1;
+    while (value >= 1024 && unit < units.length - 1) {
+        value /= 1024;
+        unit += 1;
+    }
+    return `${value.toFixed(value >= 100 ? 0 : 1)} ${units[unit]} (${bytes} bytes)`;
+}
+function percentile(values, p) {
+    const sorted = [...values].sort((a, b) => a - b);
+    return sorted[Math.min(sorted.length - 1, Math.floor((p / 100) * sorted.length))];
+}
+function formatSeconds(ms) {
+    return `${(ms / 1000).toFixed(1)}s`;
+}
+const REQUEST_LOG_PROGRESS_INTERVAL_MS = 10_000;
+function printRequestLogSummary(result, status) {
+    if (result.dryRun) {
+        const scan = result.scan;
+        console.log('[crewx] Dry run finished — nothing was written.');
+        console.log(`  Rows scanned: ${scan?.rows ?? 0}`);
+        console.log(`  Bodies: ${scan?.bodies ?? 0} (${formatBytes(scan?.bodyBytes ?? 0)})`);
+        console.log(`  Distinct bodies: ${scan?.distinctBodies ?? 0} (${formatBytes(scan?.distinctBodyBytes ?? 0)}) — what logs.db would store`);
+        console.log(`  Elapsed: ${formatSeconds(result.elapsedMs)}`);
+        return;
+    }
+    console.log('[crewx] Request-log move finished.');
+    console.log(`  Moved rows: ${result.movedRows}`);
+    console.log(`  Remaining rows in crewx.db: ${status.crewxRows}`);
+    console.log(`  logs.db request_logs: ${status.logsRows ?? 0}`);
+    console.log(`  logs.db request_log_bodies: ${status.logsBodies ?? 0}`);
+    console.log(`  logs.db size: ${formatBytes(status.logsFileBytes)} (+ wal ${formatBytes(status.logsWalBytes)})`);
+    console.log(`  crewx.db freelist pages: ${status.crewxFreelistPages}`);
+    console.log(`  Batches: ${result.batches}, bodies added: ${result.bodiesAdded}`);
+    if (result.deleteTxMs.length > 0) {
+        console.log(`  crewx.db delete transaction per batch: p50 ${percentile(result.deleteTxMs, 50).toFixed(1)} ms, p90 ${percentile(result.deleteTxMs, 90).toFixed(1)} ms, max ${result.maxDeleteTxMs.toFixed(1)} ms`);
+        console.log(`  Largest crewx.db -wal seen after a batch: ${formatBytes(result.maxWalBytes)}`);
+    }
+    console.log(`  Elapsed: ${formatSeconds(result.elapsedMs)}`);
+    if (status.crewxRows > 0) {
+        console.log(`  ${status.crewxRows} row(s) are still in crewx.db (new rows arrived or the run stopped early). Run it again.`);
+    }
+}
+async function handleRequestLogMigration(args) {
+    let parsed;
+    try {
+        parsed = parseRequestLogMigrationArgs(args);
+    }
+    catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        if (message === exports.REQUEST_LOG_MIGRATION_HELP) {
+            console.log(message);
+        }
+        else {
+            console.error(message);
+            console.error(exports.REQUEST_LOG_MIGRATION_HELP);
+            process.exitCode = 2;
+        }
+        return;
+    }
+    try {
+        const before = (0, repository_4.inspectRequestLogsMove)();
+        console.log(`[crewx] Request logs → logs.db${parsed.dryRun ? ' (dry run)' : ''}`);
+        console.log(`  crewx.db: ${before.crewxDbPath}`);
+        console.log(`  logs.db: ${before.logsDbPath}`);
+        console.log(`  Rows to move: ${before.crewxRows}`);
+        let lastPrint = Date.now();
+        const onProgress = (progress) => {
+            const now = Date.now();
+            if (now - lastPrint < REQUEST_LOG_PROGRESS_INTERVAL_MS)
+                return;
+            lastPrint = now;
+            const done = parsed.dryRun ? progress.scannedRows : progress.movedRows;
+            const remaining = Math.max(0, before.crewxRows - done);
+            console.log(`  ${parsed.dryRun ? 'scanned' : 'moved'} ${done} | remaining ~${remaining} | elapsed ${formatSeconds(progress.elapsedMs)}`);
+        };
+        const result = (0, repository_4.moveRequestLogsToLogsDb)({
+            batchRows: parsed.batchRows,
+            dryRun: parsed.dryRun,
+            onProgress,
+        });
+        printRequestLogSummary(result, (0, repository_4.inspectRequestLogsMove)());
+        process.exitCode = parsed.dryRun || result.remainingRows === 0 ? 0 : 1;
+    }
+    catch (error) {
+        console.error(error instanceof Error ? error.message : String(error));
+        process.exitCode = 1;
+    }
+}
 function parseUsageBackfillArgs(args) {
     let mode;
     let dbPath;
@@ -447,7 +599,7 @@ async function handleUsageBackfill(args) {
         return;
     }
     try {
-        const report = await (0, repository_4.runUsageBackfill)(parsed);
+        const report = await (0, repository_5.runUsageBackfill)(parsed);
         console.log(formatUsageBackfillReport(report));
         process.exitCode = report.ok ? 0 : 1;
     }

package/dist/main.js

@@ -339,6 +339,9 @@ Database:
     --dry-run                    Validate and estimate without writing
     --apply                      Consistent backup + resumable task-by-task apply
     --verify                     Check event/source invariants and parity
+  db migrate-request-logs        Move request logs from crewx.db into logs.db (bodies stored once)
+    --batch N                    Rows per batch (default 250)
+    --dry-run                    Count rows and distinct bodies without writing
 
 Desktop Shortcut (Windows only):
   shortcut status [--json]                Show current shortcut status

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@crewx/cli",
-  "version": "0.9.0-rc.139",
+  "version": "0.9.0-rc.140",
   "license": "UNLICENSED",
   "engines": {
     "node": ">=20.19.0"
@@ -25,20 +25,20 @@
     "better-sqlite3": "*",
     "dotenv": "17.2.3",
     "isomorphic-git": "1.37.1",
-    "@crewx/sdk": "0.9.0-rc.139",
-    "@crewx/knowledge-core": "0.1.17-rc.149",
-    "@crewx/memory": "0.1.23-rc.156",
-    "@crewx/search": "0.1.10-rc.135",
-    "@crewx/doc": "0.1.9-rc.132",
-    "@crewx/wbs": "0.1.10-rc.165",
-    "@crewx/workflow": "0.3.22-rc.185",
-    "@crewx/skill": "0.1.20-rc.33",
-    "@crewx/cron": "0.1.10-rc.174",
-    "@crewx/chromex": "0.1.0-rc.172",
-    "@crewx/app": "0.1.0-rc.20",
-    "@crewx/shared": "0.0.6-rc.54",
-    "@crewx/notify": "0.1.0-rc.110",
-    "@crewx/wi": "0.1.10-rc.159"
+    "@crewx/sdk": "0.9.0-rc.140",
+    "@crewx/search": "0.1.10-rc.136",
+    "@crewx/knowledge-core": "0.1.17-rc.150",
+    "@crewx/memory": "0.1.23-rc.157",
+    "@crewx/doc": "0.1.9-rc.133",
+    "@crewx/wbs": "0.1.10-rc.166",
+    "@crewx/cron": "0.1.10-rc.175",
+    "@crewx/skill": "0.1.20-rc.34",
+    "@crewx/workflow": "0.3.22-rc.186",
+    "@crewx/chromex": "0.1.0-rc.173",
+    "@crewx/notify": "0.1.0-rc.111",
+    "@crewx/shared": "0.0.6-rc.55",
+    "@crewx/wi": "0.1.10-rc.160",
+    "@crewx/app": "0.1.0-rc.21"
   },
   "devDependencies": {
     "@types/better-sqlite3": "*",