@tokscale/cli 1.0.17 → 1.0.19

package/src/graph-types.ts

@@ -6,7 +6,7 @@
 /**
  * Valid source identifiers
  */
-export type SourceType = "opencode" | "claude" | "codex" | "gemini" | "cursor";
+export type SourceType = "opencode" | "claude" | "codex" | "gemini" | "cursor" | "amp" | "droid";
 
 /**
  * Token breakdown by category

package/src/native-runner.js

@@ -0,0 +1,4 @@
+#!/usr/bin/env bun
+// Shim for bunx: when Bun loads src/ directly instead of dist/,
+// this file redirects to the TypeScript source.
+await import("./native-runner.ts");

package/src/native-runner.ts

@@ -5,57 +5,27 @@
  * This script runs in a separate process to keep the main event loop free
  * for UI rendering (e.g., spinner animation).
  * 
- * Communication: stdin (JSON input) -> stdout (JSON output)
- * No temp files needed - pure Unix IPC pattern.
+ * Communication: file (JSON input) -> stdout (JSON output)
  */
 
 import nativeCore from "@tokscale/core";
-
-const MAX_INPUT_SIZE = 50 * 1024 * 1024; // 50MB
-const STDIN_TIMEOUT_MS = 30_000; // 30s
+import { readFileSync } from "node:fs";
 
 interface NativeRunnerRequest {
   method: string;
   args: unknown[];
 }
 
-async function readStdinWithLimits(): Promise<string> {
-  const chunks: Buffer[] = [];
-  let totalSize = 0;
-  let timeoutId: ReturnType<typeof setTimeout> | null = null;
-
-  const timeoutPromise = new Promise<never>((_, reject) => {
-    timeoutId = setTimeout(() => reject(new Error(`Stdin read timed out after ${STDIN_TIMEOUT_MS}ms`)), STDIN_TIMEOUT_MS);
-  });
-
-  const readPromise = (async () => {
-    for await (const chunk of process.stdin) {
-      const buf = Buffer.from(chunk as ArrayBuffer);
-      totalSize += buf.length;
-      if (totalSize > MAX_INPUT_SIZE) {
-        throw new Error(`Input exceeds maximum size of ${MAX_INPUT_SIZE} bytes (${Math.round(MAX_INPUT_SIZE / 1024 / 1024)}MB)`);
-      }
-      chunks.push(buf);
-    }
-    return Buffer.concat(chunks).toString("utf-8");
-  })();
-
-  try {
-    const result = await Promise.race([readPromise, timeoutPromise]);
-    return result;
-  } finally {
-    if (timeoutId) clearTimeout(timeoutId);
-  }
-}
-
 async function main() {
-  const input = await readStdinWithLimits();
+  const inputFile = process.argv[2];
   
-  if (!input.trim()) {
-    process.stderr.write(JSON.stringify({ error: "No input received" }));
+  if (!inputFile) {
+    process.stderr.write(JSON.stringify({ error: "No input file provided" }));
     process.exit(1);
   }
   
+  const input = readFileSync(inputFile, "utf-8");
+  
   let request: NativeRunnerRequest;
   try {
     request = JSON.parse(input) as NativeRunnerRequest;
@@ -76,16 +46,16 @@ async function main() {
       result = nativeCore.parseLocalSources(args[0] as Parameters<typeof nativeCore.parseLocalSources>[0]);
       break;
     case "finalizeReport":
-      result = nativeCore.finalizeReport(args[0] as Parameters<typeof nativeCore.finalizeReport>[0]);
+      result = await nativeCore.finalizeReport(args[0] as Parameters<typeof nativeCore.finalizeReport>[0]);
       break;
     case "finalizeMonthlyReport":
-      result = nativeCore.finalizeMonthlyReport(args[0] as Parameters<typeof nativeCore.finalizeMonthlyReport>[0]);
+      result = await nativeCore.finalizeMonthlyReport(args[0] as Parameters<typeof nativeCore.finalizeMonthlyReport>[0]);
       break;
     case "finalizeGraph":
-      result = nativeCore.finalizeGraph(args[0] as Parameters<typeof nativeCore.finalizeGraph>[0]);
+      result = await nativeCore.finalizeGraph(args[0] as Parameters<typeof nativeCore.finalizeGraph>[0]);
       break;
-    case "generateGraphWithPricing":
-      result = nativeCore.generateGraphWithPricing(args[0] as Parameters<typeof nativeCore.generateGraphWithPricing>[0]);
+    case "finalizeReportAndGraph":
+      result = await nativeCore.finalizeReportAndGraph(args[0] as Parameters<typeof nativeCore.finalizeReportAndGraph>[0]);
       break;
     default:
       throw new Error(`Unknown method: ${method}`);

package/src/native.ts

@@ -2,48 +2,19 @@
  * Native module loader for Rust core
  *
  * Exposes all Rust functions with proper TypeScript types.
- * Falls back to TypeScript implementations when native module is unavailable.
+ * Native module is REQUIRED - no TypeScript fallback.
  */
 
-import type { PricingEntry } from "./pricing.js";
 import type {
   TokenContributionData,
   GraphOptions as TSGraphOptions,
   SourceType,
 } from "./graph-types.js";
-import {
-  parseLocalSources as parseLocalSourcesTS,
-  type ParsedMessages as TSParsedMessages,
-  type UnifiedMessage,
-} from "./sessions/index.js";
-import {
-  generateModelReport as generateModelReportTS,
-  generateMonthlyReport as generateMonthlyReportTS,
-  generateGraphData as generateGraphDataTS,
-} from "./sessions/reports.js";
-import { readCursorMessagesFromCache } from "./cursor.js";
 
 // =============================================================================
 // Types matching Rust exports
 // =============================================================================
 
-interface NativeGraphOptions {
-  homeDir?: string;
-  sources?: string[];
-  since?: string;
-  until?: string;
-  year?: string;
-  threads?: number;
-}
-
-interface NativeScanStats {
-  opencodeFiles: number;
-  claudeFiles: number;
-  codexFiles: number;
-  geminiFiles: number;
-  totalFiles: number;
-}
-
 interface NativeTokenBreakdown {
   input: number;
   output: number;
@@ -109,21 +80,9 @@ interface NativeGraphResult {
   contributions: NativeDailyContribution[];
 }
 
-// Types for pricing-aware APIs
-interface NativePricingEntry {
-  modelId: string;
-  pricing: {
-    inputCostPerToken: number;
-    outputCostPerToken: number;
-    cacheReadInputTokenCost?: number;
-    cacheCreationInputTokenCost?: number;
-  };
-}
-
 interface NativeReportOptions {
   homeDir?: string;
   sources?: string[];
-  pricing: NativePricingEntry[];
   since?: string;
   until?: string;
   year?: string;
@@ -192,6 +151,8 @@ interface NativeParsedMessages {
   claudeCount: number;
   codexCount: number;
   geminiCount: number;
+  ampCount: number;
+  droidCount?: number;
   processingTimeMs: number;
 }
 
@@ -206,7 +167,6 @@ interface NativeLocalParseOptions {
 interface NativeFinalizeReportOptions {
   homeDir?: string;
   localMessages: NativeParsedMessages;
-  pricing: NativePricingEntry[];
   includeCursor: boolean;
   since?: string;
   until?: string;
@@ -216,12 +176,6 @@ interface NativeFinalizeReportOptions {
 interface NativeCore {
   version(): string;
   healthCheck(): string;
-  generateGraph(options: NativeGraphOptions): NativeGraphResult;
-  generateGraphWithPricing(options: NativeReportOptions): NativeGraphResult;
-  scanSessions(homeDir?: string, sources?: string[]): NativeScanStats;
-  getModelReport(options: NativeReportOptions): NativeModelReport;
-  getMonthlyReport(options: NativeReportOptions): NativeMonthlyReport;
-  // Two-phase processing (parallel optimization)
   parseLocalSources(options: NativeLocalParseOptions): NativeParsedMessages;
   finalizeReport(options: NativeFinalizeReportOptions): NativeModelReport;
   finalizeMonthlyReport(options: NativeFinalizeReportOptions): NativeMonthlyReport;
@@ -236,7 +190,11 @@ let nativeCore: NativeCore | null = null;
 let loadError: Error | null = null;
 
 try {
-  nativeCore = await import("@tokscale/core").then((m) => m.default || m);
+  // Type assertion needed because dynamic import returns module namespace
+  // nativeCore.version() is called directly, async functions go through subprocess
+  nativeCore = await import("@tokscale/core").then(
+    (m) => (m.default || m) as unknown as NativeCore
+  );
 } catch (e) {
   loadError = e as Error;
 }
@@ -252,13 +210,6 @@ export function isNativeAvailable(): boolean {
   return nativeCore !== null;
 }
 
-/**
- * Get native module load error (if any)
- */
-export function getNativeLoadError(): Error | null {
-  return loadError;
-}
-
 /**
  * Get native module version
  */
@@ -266,33 +217,6 @@ export function getNativeVersion(): string | null {
   return nativeCore?.version() ?? null;
 }
 
-/**
- * Scan sessions using native module
- */
-export function scanSessionsNative(homeDir?: string, sources?: string[]): NativeScanStats | null {
-  if (!nativeCore) {
-    return null;
-  }
-  return nativeCore.scanSessions(homeDir, sources);
-}
-
-// =============================================================================
-// Graph generation
-// =============================================================================
-
-/**
- * Convert TypeScript graph options to native format
- */
-function toNativeOptions(options: TSGraphOptions): NativeGraphOptions {
-  return {
-    homeDir: undefined,
-    sources: options.sources,
-    since: options.since,
-    until: options.until,
-    year: options.year,
-  };
-}
-
 /**
  * Convert native result to TypeScript format
  */
@@ -358,22 +282,6 @@ function fromNativeResult(result: NativeGraphResult): TokenContributionData {
   };
 }
 
-/**
- * Generate graph data using native module (without pricing - uses embedded costs)
- * @deprecated Use generateGraphWithPricing instead
- */
-export function generateGraphNative(options: TSGraphOptions = {}): TokenContributionData {
-  if (!nativeCore) {
-    throw new Error("Native module not available: " + (loadError?.message || "unknown error"));
-  }
-
-  const nativeOptions = toNativeOptions(options);
-  const result = nativeCore.generateGraph(nativeOptions);
-  return fromNativeResult(result);
-}
-
-
-
 // =============================================================================
 // Reports
 // =============================================================================
@@ -442,6 +350,8 @@ export interface ParsedMessages {
   claudeCount: number;
   codexCount: number;
   geminiCount: number;
+  ampCount: number;
+  droidCount: number;
   processingTimeMs: number;
 }
 
@@ -450,13 +360,10 @@ export interface LocalParseOptions {
   since?: string;
   until?: string;
   year?: string;
-  /** Force TypeScript fallback even when native module is available (needed for agent field) */
-  forceTypescript?: boolean;
 }
 
 export interface FinalizeOptions {
   localMessages: ParsedMessages;
-  pricing: PricingEntry[];
   includeCursor: boolean;
   since?: string;
   until?: string;
@@ -471,6 +378,9 @@ export interface FinalizeOptions {
 
 import { fileURLToPath } from "node:url";
 import { dirname, join } from "node:path";
+import { writeFileSync, unlinkSync, mkdirSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { randomUUID } from "node:crypto";
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
@@ -489,7 +399,6 @@ const MAX_OUTPUT_BYTES = parseInt(
 );
 
 interface BunSubprocess {
-  stdin: { write: (data: string) => void; end: () => void };
   stdout: { text: () => Promise<string> };
   stderr: { text: () => Promise<string> };
   exited: Promise<number>;
@@ -499,7 +408,6 @@ interface BunSubprocess {
 }
 
 interface BunSpawnOptions {
-  stdin: string;
   stdout: string;
   stderr: string;
 }
@@ -518,16 +426,22 @@ async function runInSubprocess<T>(method: string, args: unknown[]): Promise<T> {
   const runnerPath = join(__dirname, "native-runner.js");
   const input = JSON.stringify({ method, args });
 
+  const tmpDir = join(tmpdir(), "tokscale");
+  mkdirSync(tmpDir, { recursive: true });
+  const inputFile = join(tmpDir, `input-${randomUUID()}.json`);
+  
+  writeFileSync(inputFile, input, "utf-8");
+
   const BunGlobal = (globalThis as Record<string, unknown>).Bun as BunGlobalType;
 
   let proc: BunSubprocess;
   try {
-    proc = BunGlobal.spawn([process.execPath, runnerPath], {
-      stdin: "pipe",
+    proc = BunGlobal.spawn([process.execPath, runnerPath, inputFile], {
       stdout: "pipe",
       stderr: "pipe",
     });
   } catch (e) {
+    unlinkSync(inputFile);
     throw new Error(`Failed to spawn subprocess: ${(e as Error).message}`);
   }
 
@@ -539,6 +453,7 @@ async function runInSubprocess<T>(method: string, args: unknown[]): Promise<T> {
   const cleanup = async () => {
     if (timeoutId) clearTimeout(timeoutId);
     if (sigkillId) clearTimeout(sigkillId);
+    try { unlinkSync(inputFile); } catch {}
     if (aborted) {
       safeKill(proc, "SIGKILL");
       await proc.exited.catch(() => {});
@@ -551,9 +466,6 @@ async function runInSubprocess<T>(method: string, args: unknown[]): Promise<T> {
   };
 
   try {
-    proc.stdin.write(input);
-    proc.stdin.end();
-
     const stdoutChunks: Uint8Array[] = [];
     const stderrChunks: Uint8Array[] = [];
     let stdoutBytes = 0;
@@ -641,36 +553,8 @@ async function runInSubprocess<T>(method: string, args: unknown[]): Promise<T> {
 }
 
 export async function parseLocalSourcesAsync(options: LocalParseOptions): Promise<ParsedMessages> {
-  // Use TypeScript fallback when native module is not available or when explicitly requested
-  if (!isNativeAvailable() || options.forceTypescript) {
-    const result = parseLocalSourcesTS({
-      sources: options.sources,
-      since: options.since,
-      until: options.until,
-      year: options.year,
-    });
-
-    return {
-      messages: result.messages.map((msg) => ({
-        source: msg.source,
-        modelId: msg.modelId,
-        providerId: msg.providerId,
-        timestamp: msg.timestamp,
-        date: msg.date,
-        input: msg.tokens.input,
-        output: msg.tokens.output,
-        cacheRead: msg.tokens.cacheRead,
-        cacheWrite: msg.tokens.cacheWrite,
-        reasoning: msg.tokens.reasoning,
-        sessionId: msg.sessionId,
-        agent: msg.agent,
-      })),
-      opencodeCount: result.opencodeCount,
-      claudeCount: result.claudeCount,
-      codexCount: result.codexCount,
-      geminiCount: result.geminiCount,
-      processingTimeMs: result.processingTimeMs,
-    };
+  if (!isNativeAvailable()) {
+    throw new Error("Native module required. Run: bun run build:core");
   }
 
   const nativeOptions: NativeLocalParseOptions = {
@@ -684,52 +568,14 @@ export async function parseLocalSourcesAsync(options: LocalParseOptions): Promis
   return runInSubprocess<ParsedMessages>("parseLocalSources", [nativeOptions]);
 }
 
-function buildMessagesForFallback(options: FinalizeOptions): UnifiedMessage[] {
-  const messages: UnifiedMessage[] = options.localMessages.messages.map((msg) => ({
-    source: msg.source,
-    modelId: msg.modelId,
-    providerId: msg.providerId,
-    sessionId: msg.sessionId,
-    timestamp: msg.timestamp,
-    date: msg.date,
-    tokens: {
-      input: msg.input,
-      output: msg.output,
-      cacheRead: msg.cacheRead,
-      cacheWrite: msg.cacheWrite,
-      reasoning: msg.reasoning,
-    },
-    cost: 0,
-    agent: msg.agent,
-  }));
-
-  if (options.includeCursor) {
-    const cursorMessages = readCursorMessagesFromCache();
-    for (const cursor of cursorMessages) {
-      const inRange =
-        (!options.year || cursor.date.startsWith(options.year)) &&
-        (!options.since || cursor.date >= options.since) &&
-        (!options.until || cursor.date <= options.until);
-      if (inRange) {
-        messages.push(cursor);
-      }
-    }
-  }
-
-  return messages;
-}
-
 export async function finalizeReportAsync(options: FinalizeOptions): Promise<ModelReport> {
   if (!isNativeAvailable()) {
-    const startTime = performance.now();
-    const messages = buildMessagesForFallback(options);
-    return generateModelReportTS(messages, options.pricing, startTime);
+    throw new Error("Native module required. Run: bun run build:core");
   }
 
   const nativeOptions: NativeFinalizeReportOptions = {
     homeDir: undefined,
     localMessages: options.localMessages,
-    pricing: options.pricing,
     includeCursor: options.includeCursor,
     since: options.since,
     until: options.until,
@@ -741,15 +587,12 @@ export async function finalizeReportAsync(options: FinalizeOptions): Promise<Mod
 
 export async function finalizeMonthlyReportAsync(options: FinalizeOptions): Promise<MonthlyReport> {
   if (!isNativeAvailable()) {
-    const startTime = performance.now();
-    const messages = buildMessagesForFallback(options);
-    return generateMonthlyReportTS(messages, options.pricing, startTime);
+    throw new Error("Native module required. Run: bun run build:core");
   }
 
   const nativeOptions: NativeFinalizeReportOptions = {
     homeDir: undefined,
     localMessages: options.localMessages,
-    pricing: options.pricing,
     includeCursor: options.includeCursor,
     since: options.since,
     until: options.until,
@@ -761,15 +604,12 @@ export async function finalizeMonthlyReportAsync(options: FinalizeOptions): Prom
 
 export async function finalizeGraphAsync(options: FinalizeOptions): Promise<TokenContributionData> {
   if (!isNativeAvailable()) {
-    const startTime = performance.now();
-    const messages = buildMessagesForFallback(options);
-    return generateGraphDataTS(messages, options.pricing, startTime);
+    throw new Error("Native module required. Run: bun run build:core");
   }
 
   const nativeOptions: NativeFinalizeReportOptions = {
     homeDir: undefined,
     localMessages: options.localMessages,
-    pricing: options.pricing,
     includeCursor: options.includeCursor,
     since: options.since,
     until: options.until,
@@ -780,33 +620,33 @@ export async function finalizeGraphAsync(options: FinalizeOptions): Promise<Toke
   return fromNativeResult(result);
 }
 
-export async function generateGraphWithPricingAsync(
-  options: TSGraphOptions & { pricing: PricingEntry[] }
-): Promise<TokenContributionData> {
-  // Use TypeScript fallback when native module is not available
-  if (!isNativeAvailable()) {
-    const startTime = performance.now();
-
-    // Parse local sources using TS fallback
-    const parsed = parseLocalSourcesTS({
-      sources: options.sources,
-      since: options.since,
-      until: options.until,
-      year: options.year,
-    });
+export interface ReportAndGraph {
+  report: ModelReport;
+  graph: TokenContributionData;
+}
+
+interface NativeReportAndGraph {
+  report: NativeModelReport;
+  graph: NativeGraphResult;
+}
 
-    return generateGraphDataTS(parsed.messages, options.pricing, startTime);
+export async function finalizeReportAndGraphAsync(options: FinalizeOptions): Promise<ReportAndGraph> {
+  if (!isNativeAvailable()) {
+    throw new Error("Native module required. Run: bun run build:core");
   }
 
-  const nativeOptions: NativeReportOptions = {
+  const nativeOptions: NativeFinalizeReportOptions = {
     homeDir: undefined,
-    sources: options.sources,
-    pricing: options.pricing,
+    localMessages: options.localMessages,
+    includeCursor: options.includeCursor,
     since: options.since,
     until: options.until,
     year: options.year,
   };
 
-  const result = await runInSubprocess<NativeGraphResult>("generateGraphWithPricing", [nativeOptions]);
-  return fromNativeResult(result);
+  const result = await runInSubprocess<NativeReportAndGraph>("finalizeReportAndGraph", [nativeOptions]);
+  return {
+    report: result.report,
+    graph: fromNativeResult(result.graph),
+  };
 }

package/src/sessions/types.ts

@@ -22,7 +22,7 @@ export interface UnifiedMessage {
   agent?: string;
 }
 
-export type SourceType = "opencode" | "claude" | "codex" | "gemini" | "cursor";
+export type SourceType = "opencode" | "claude" | "codex" | "gemini" | "cursor" | "amp" | "droid";
 
 /**
  * Convert Unix milliseconds timestamp to YYYY-MM-DD date string

package/src/submit.ts

@@ -5,11 +5,8 @@
 
 import pc from "picocolors";
 import { loadCredentials, getApiBaseUrl } from "./credentials.js";
-import { PricingFetcher } from "./pricing.js";
-import {
-  isNativeAvailable,
-  generateGraphWithPricingAsync,
-} from "./native.js";
+import { parseLocalSourcesAsync, finalizeReportAndGraphAsync, type ParsedMessages } from "./native.js";
+import { syncCursorCache, loadCursorCredentials } from "./cursor.js";
 import type { TokenContributionData } from "./graph-types.js";
 import { formatCurrency } from "./table.js";
 
@@ -19,6 +16,8 @@ interface SubmitOptions {
   codex?: boolean;
   gemini?: boolean;
   cursor?: boolean;
+  amp?: boolean;
+  droid?: boolean;
   since?: string;
   until?: string;
   year?: string;
@@ -44,7 +43,7 @@ interface SubmitResponse {
   details?: string[];
 }
 
-type SourceType = "opencode" | "claude" | "codex" | "gemini" | "cursor";
+type SourceType = "opencode" | "claude" | "codex" | "gemini" | "cursor" | "amp" | "droid";
 
 /**
  * Submit command - sends usage data to the platform
@@ -58,24 +57,13 @@ export async function submit(options: SubmitOptions = {}): Promise<void> {
     process.exit(1);
   }
 
-  // Step 2: Log native module status (TS fallback available)
-  if (!isNativeAvailable()) {
-    console.log(pc.yellow("\n  Note: Using TypeScript fallback (native module not available)"));
-    console.log(pc.gray("  Run 'bun run build:core' for faster processing.\n"));
-  }
-
   console.log(pc.cyan("\n  Tokscale - Submit Usage Data\n"));
 
-  // Step 3: Generate graph data
   console.log(pc.gray("  Scanning local session data..."));
 
-  const fetcher = new PricingFetcher();
-  await fetcher.fetchPricing();
-  const pricingEntries = fetcher.toPricingEntries();
-
-  // Determine sources
-  const hasFilter = options.opencode || options.claude || options.codex || options.gemini || options.cursor;
+  const hasFilter = options.opencode || options.claude || options.codex || options.gemini || options.cursor || options.amp || options.droid;
   let sources: SourceType[] | undefined;
+  let includeCursor = true;
   if (hasFilter) {
     sources = [];
     if (options.opencode) sources.push("opencode");
@@ -83,17 +71,43 @@ export async function submit(options: SubmitOptions = {}): Promise<void> {
     if (options.codex) sources.push("codex");
     if (options.gemini) sources.push("gemini");
     if (options.cursor) sources.push("cursor");
+    if (options.amp) sources.push("amp");
+    if (options.droid) sources.push("droid");
+    includeCursor = sources.includes("cursor");
   }
 
+  // Filter out cursor from local sources (it's handled separately via sync)
+  const localSources = sources?.filter((s): s is Exclude<SourceType, "cursor"> => s !== "cursor");
+
   let data: TokenContributionData;
   try {
-    data = await generateGraphWithPricingAsync({
-      sources,
-      pricing: pricingEntries,
+    // Two-phase processing (same as TUI) for consistency:
+    // Phase 1: Parse local sources + sync cursor in parallel
+    const [localMessages, cursorSync] = await Promise.all([
+      parseLocalSourcesAsync({
+        sources: localSources,
+        since: options.since,
+        until: options.until,
+        year: options.year,
+      }),
+      includeCursor && loadCursorCredentials()
+        ? syncCursorCache()
+        : Promise.resolve({ synced: false, rows: 0 }),
+    ]);
+
+    // Phase 2: Finalize with pricing (combines local + cursor)
+    // Single subprocess call ensures consistent pricing for both report and graph
+    const { report, graph } = await finalizeReportAndGraphAsync({
+      localMessages,
+      includeCursor: includeCursor && cursorSync.synced,
       since: options.since,
       until: options.until,
       year: options.year,
     });
+
+    // Use graph structure for submission, report's cost for display
+    data = graph;
+    data.summary.totalCost = report.totalCost;
   } catch (error) {
     console.error(pc.red(`\n  Error generating data: ${(error as Error).message}\n`));
     process.exit(1);