@crewx/cli 0.8.0-rc.66

package/scripts/backfill-tokens.js

@@ -0,0 +1,218 @@
+#!/usr/bin/env node
+/**
+ * backfill-tokens.js
+ *
+ * One-time migration script to backfill input_tokens, output_tokens, cost_usd
+ * for existing tasks whose logs already contain usage data but were recorded
+ * before extractUsageFromLogs() was introduced.
+ *
+ * Usage:
+ *   node packages/cli/scripts/backfill-tokens.js [--dry-run] [db-path]
+ *
+ * DB resolution order:
+ *   1. Positional argument
+ *   2. CREWX_DB env var (CREWX_TRACES_DB kept for backward compat)
+ *   3. .crewx/crewx.db (local, if exists)
+ *   4. ~/.crewx/crewx.db (global)
+ */
+
+const { Database } = require('node-sqlite3-wasm');
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+
+// ─── Pricing table (mirrors packages/sdk/src/config/pricing.ts) ─────────────
+
+const MODEL_PRICING = {
+  'claude-opus':            { input: 15,    output: 75   },
+  'opus':                   { input: 15,    output: 75   },
+  'claude-sonnet':          { input: 3,     output: 15   },
+  'sonnet':                 { input: 3,     output: 15   },
+  'claude-haiku':           { input: 0.25,  output: 1.25 },
+  'haiku':                  { input: 0.25,  output: 1.25 },
+  'gemini-3-pro-preview':   { input: 1.25,  output: 5    },
+  'gemini-2-flash':         { input: 0.075, output: 0.30 },
+  'gemini-flash':           { input: 0.075, output: 0.30 },
+  'default':                { input: 1,     output: 1    },
+};
+
+function getPricing(model) {
+  const fallback = MODEL_PRICING['default'];
+  if (!model) return fallback;
+
+  const normalized = model.toLowerCase();
+
+  if (MODEL_PRICING[normalized]) return MODEL_PRICING[normalized];
+
+  for (const key of Object.keys(MODEL_PRICING)) {
+    if (normalized.includes(key)) return MODEL_PRICING[key];
+  }
+
+  return fallback;
+}
+
+function calculateCost(inputTokens, outputTokens, model) {
+  const pricing = getPricing(model);
+  return (inputTokens * pricing.input + outputTokens * pricing.output) / 1_000_000;
+}
+
+// ─── Usage extraction (mirrors tracing.service.ts extractUsageFromLogs) ──────
+
+function extractUsageFromLogs(logs) {
+  if (!logs || logs.length === 0) return null;
+
+  let found = null;
+
+  for (const entry of logs) {
+    if (entry.level !== 'stdout') continue;
+
+    let parsed;
+    try {
+      parsed = JSON.parse(entry.message);
+    } catch {
+      continue;
+    }
+
+    // Claude stream-json: {"type":"result","subtype":"success","usage":{...}}
+    if (
+      parsed.type === 'result' &&
+      parsed.subtype === 'success' &&
+      parsed.usage &&
+      typeof parsed.usage === 'object'
+    ) {
+      const u = parsed.usage;
+      if (typeof u.input_tokens === 'number' && typeof u.output_tokens === 'number') {
+        found = { inputTokens: u.input_tokens, outputTokens: u.output_tokens };
+      }
+    }
+
+    // Codex experimental-json: {"type":"turn.completed","usage":{...}}
+    if (
+      parsed.type === 'turn.completed' &&
+      parsed.usage &&
+      typeof parsed.usage === 'object'
+    ) {
+      const u = parsed.usage;
+      if (typeof u.input_tokens === 'number' && typeof u.output_tokens === 'number') {
+        found = { inputTokens: u.input_tokens, outputTokens: u.output_tokens };
+      }
+    }
+  }
+
+  return found;
+}
+
+// ─── DB path resolution ───────────────────────────────────────────────────────
+
+function resolveDbPaths(argPath) {
+  const paths = [];
+
+  if (argPath) {
+    paths.push(path.resolve(argPath));
+    return paths; // explicit path: only use that one
+  }
+
+  if (process.env.CREWX_TRACES_DB || process.env.CREWX_DB) {
+    paths.push(path.resolve(process.env.CREWX_TRACES_DB || process.env.CREWX_DB));
+  }
+
+  const localDb = path.join(process.cwd(), '.crewx', 'crewx.db');
+  if (fs.existsSync(localDb)) {
+    paths.push(localDb);
+  }
+
+  const globalDb = path.join(os.homedir(), '.crewx', 'crewx.db');
+  if (fs.existsSync(globalDb) && globalDb !== localDb) {
+    paths.push(globalDb);
+  }
+
+  return [...new Set(paths)]; // deduplicate
+}
+
+// ─── Backfill logic ───────────────────────────────────────────────────────────
+
+function backfillDb(dbPath, dryRun) {
+  if (!fs.existsSync(dbPath)) {
+    console.log(`  [SKIP] DB not found: ${dbPath}`);
+    return { processed: 0, updated: 0, skipped: 0 };
+  }
+
+  console.log(`\nProcessing: ${dbPath}${dryRun ? ' (dry-run)' : ''}`);
+
+  const db = new Database(dbPath, { readOnly: dryRun });
+
+  // Only target tasks where tokens are still 0 and logs exist
+  const tasks = db.all(`
+    SELECT id, model, logs
+    FROM tasks
+    WHERE (input_tokens = 0 OR input_tokens IS NULL)
+      AND logs IS NOT NULL
+      AND logs != '[]'
+      AND logs != ''
+  `);
+
+  console.log(`  Found ${tasks.length} tasks with tokens=0 and non-empty logs`);
+
+  let updated = 0;
+  let skipped = 0;
+
+  for (const task of tasks) {
+    let logs;
+    try {
+      logs = JSON.parse(task.logs);
+    } catch {
+      skipped++;
+      continue;
+    }
+
+    const usage = extractUsageFromLogs(logs);
+    if (!usage) {
+      skipped++;
+      continue;
+    }
+
+    const costUsd = calculateCost(usage.inputTokens, usage.outputTokens, task.model);
+
+    if (dryRun) {
+      console.log(`  [DRY] id=${task.id} model=${task.model ?? 'unknown'} input=${usage.inputTokens} output=${usage.outputTokens} cost=$${costUsd.toFixed(6)}`);
+    } else {
+      db.run('UPDATE tasks SET input_tokens=?, output_tokens=?, cost_usd=? WHERE id=?', [usage.inputTokens, usage.outputTokens, costUsd, task.id]);
+    }
+
+    updated++;
+  }
+
+  db.close();
+
+  console.log(`  Updated: ${updated}, Skipped (no usage found): ${skipped}`);
+  return { processed: tasks.length, updated, skipped };
+}
+
+// ─── Main ─────────────────────────────────────────────────────────────────────
+
+const args = process.argv.slice(2);
+const dryRun = args.includes('--dry-run');
+const dbArg = args.find(a => !a.startsWith('--'));
+
+const dbPaths = resolveDbPaths(dbArg);
+
+if (dbPaths.length === 0) {
+  console.error('No crewx.db found. Pass a path explicitly or set CREWX_DB.');
+  process.exit(1);
+}
+
+console.log(`backfill-tokens ${dryRun ? '[DRY RUN] ' : ''}— ${dbPaths.length} DB(s) to process`);
+
+let totalUpdated = 0;
+let totalSkipped = 0;
+
+for (const dbPath of dbPaths) {
+  const { updated, skipped } = backfillDb(dbPath, dryRun);
+  totalUpdated += updated;
+  totalSkipped += skipped;
+}
+
+console.log(`\nDone. Total updated: ${totalUpdated}, skipped: ${totalSkipped}`);
+if (dryRun) {
+  console.log('(Dry-run mode — no changes were written)');
+}

package/scripts/postbuild-cli.mjs

@@ -0,0 +1,88 @@
+#!/usr/bin/env node
+/**
+ * Ensure the CLI build artifact has an executable shebang after compilation.
+ * Works both inside the monorepo and when the package is installed from npm.
+ */
+import { chmodSync, existsSync, readFileSync, writeFileSync, cpSync, rmSync, readdirSync } from 'fs';
+import { dirname, join } from 'path';
+import { fileURLToPath } from 'url';
+
+const scriptDir = dirname(fileURLToPath(import.meta.url));
+const packageRoot = dirname(scriptDir);
+const candidateTemplateSources = [
+  join(dirname(packageRoot), 'templates'),
+  join(dirname(dirname(packageRoot)), 'templates'),
+  join(packageRoot, 'templates'),
+];
+const sourceTemplatesDir = candidateTemplateSources.find(candidate => existsSync(candidate));
+const targetTemplatesDir = join(packageRoot, 'templates');
+
+try {
+  if (sourceTemplatesDir && sourceTemplatesDir !== targetTemplatesDir) {
+    rmSync(targetTemplatesDir, { recursive: true, force: true });
+    cpSync(sourceTemplatesDir, targetTemplatesDir, { recursive: true });
+    console.log(`✅ Synced templates to ${targetTemplatesDir}`);
+  } else if (!sourceTemplatesDir && !existsSync(targetTemplatesDir)) {
+    console.warn('⚠️ Templates directory not found in monorepo; packaged CLI may rely on CDN fallback.');
+  }
+} catch (err) {
+  console.warn(`⚠️ Failed to sync templates directory: ${err instanceof Error ? err.message : err}`);
+}
+
+const candidatePaths = [
+  join(packageRoot, 'dist', 'main.js'),
+  join(packageRoot, '..', '..', 'dist', 'main.js')
+];
+
+const targetPath = candidatePaths.find(p => existsSync(p));
+
+if (!targetPath) {
+  console.warn('⚠️ postbuild skipped: dist/main.js not found (expected during migration)');
+  process.exit(0);
+}
+
+let content = readFileSync(targetPath, 'utf8');
+const lines = content.split('\n');
+if (lines[0] === '#!/usr/bin/env node' && lines[1] === '#!/usr/bin/env node') {
+  lines.shift();
+  content = lines.join('\n');
+  writeFileSync(targetPath, content);
+  console.log('✅ Removed duplicate shebang');
+} else if (!content.startsWith('#!/usr/bin/env node')) {
+  writeFileSync(targetPath, '#!/usr/bin/env node\n' + content);
+  console.log('✅ Added shebang to dist/main.js');
+} else {
+  console.log('✅ Shebang already present');
+}
+
+chmodSync(targetPath, 0o755);
+console.log(`✅ Execute permission set (${targetPath})`);
+
+// Ensure dist/ is treated as CJS even when root package.json has "type": "module".
+// Without this, global installs fail because Node resolves module type from the
+// nearest parent package.json — which is the root one after npm strips the
+// packages/cli/package.json during pack.
+const distDir = join(packageRoot, 'dist');
+const distPkgPath = join(distDir, 'package.json');
+writeFileSync(distPkgPath, JSON.stringify({ type: 'commonjs' }, null, 2) + '\n');
+console.log('✅ Created dist/package.json with "type": "commonjs"');
+
+// Rewrite workspace-local SDK paths to npm package names.
+// nest build resolves `file:../sdk` as relative paths (e.g., ../../sdk/dist/api),
+// which break when installed globally from npm. Replace with package names.
+const distJsFiles = readdirSync(distDir).filter(f => f.endsWith('.js'));
+for (const file of distJsFiles) {
+  const filePath = join(distDir, file);
+  let src = readFileSync(filePath, 'utf8');
+  const before = src;
+  src = src.replace(/require\("\.\.\/\.\.\/sdk\/dist\/api"\)/g, 'require("@crewx/sdk/api")');
+  src = src.replace(/require\("\.\.\/\.\.\/sdk\/dist\/tools"\)/g, 'require("@crewx/sdk/tools")');
+  src = src.replace(/require\("\.\.\/\.\.\/sdk\/dist\/internal"\)/g, 'require("@crewx/sdk/internal")');
+  src = src.replace(/require\("\.\.\/\.\.\/sdk\/dist\/core\/env-defaults"\)/g, 'require("@crewx/sdk/core/env-defaults")');
+  src = src.replace(/require\("\.\.\/\.\.\/sdk\/dist"\)/g, 'require("@crewx/sdk")');
+  src = src.replace(/require\("\.\.\/\.\.\/sdk\/dist\/([^"]+)"\)/g, 'require("@crewx/sdk/dist/$1")');
+  if (src !== before) {
+    writeFileSync(filePath, src);
+    console.log(`✅ Rewrote SDK paths in ${file}`);
+  }
+}

package/scripts/postinstall-cli.mjs

@@ -0,0 +1,30 @@
+#!/usr/bin/env node
+/**
+ * postinstall hook used after npm install to ensure the CrewX CLI binary is executable.
+ * Works in both workspace and published package contexts.
+ */
+import { chmodSync, existsSync } from 'fs';
+import { dirname, join } from 'path';
+import { fileURLToPath } from 'url';
+
+const scriptDir = dirname(fileURLToPath(import.meta.url));
+const packageRoot = dirname(scriptDir);
+
+const candidatePaths = [
+  join(packageRoot, 'dist', 'main.js'),
+  join(packageRoot, '..', '..', 'dist', 'main.js')
+];
+
+const targetPath = candidatePaths.find(p => existsSync(p));
+
+if (!targetPath) {
+  console.warn('⚠️ postinstall skipped: dist/main.js not found (expected during migration)');
+  process.exit(0);
+}
+
+try {
+  chmodSync(targetPath, 0o755);
+  console.log(`✅ Execute permission set for crewx binary (${targetPath})`);
+} catch (err) {
+  console.warn('⚠️ postinstall could not set execute permission:', err.message);
+}