@yalehwang/cloudpit 2.0.0 → 3.0.0

package/server/modules/cli/asset-migration-store.ts

@@ -0,0 +1,228 @@
+import fs from 'node:fs';
+import path from 'node:path';
+
+import Database from 'better-sqlite3';
+
+/**
+ * One app-owned persisted document that may embed a legacy asset path.
+ *
+ * `text` is the whole document (a JSON file's contents or one SQLite cell), so
+ * the migration service can replace the path prefix and write the document back
+ * as a unit. `handle` is opaque to the service and names what to write:
+ * rewriting a SQLite row needs its key columns, and the service must not parse
+ * a human-readable location string to recover them.
+ */
+export type PersistedAssetPathDocument = {
+  /** Human-readable identity used in reports, e.g. `scheduled_messages[id=7].options`. */
+  location: string;
+  /** Current full text of the document. */
+  text: string;
+  /** Where the rewritten text belongs. */
+  handle: AssetPathDocumentHandle;
+};
+
+/** Where one {@link PersistedAssetPathDocument} is stored. */
+export type AssetPathDocumentHandle =
+  | { kind: 'json-file'; filePath: string }
+  | {
+      kind: 'sqlite';
+      table: string;
+      column: string;
+      keyColumns: string[];
+      keyValues: Array<string | number>;
+    };
+
+/** One document plus the text that should replace its current contents. */
+export type AssetPathDocumentUpdate = {
+  document: PersistedAssetPathDocument;
+  text: string;
+};
+
+/**
+ * The app-owned stores whose documents may carry attachment paths.
+ *
+ * The migration service reads every document, rewrites the ones containing the
+ * legacy root, and applies the whole batch through {@link writeDocuments}.
+ * Backups are taken before the first write so {@link restore} can put the
+ * stores back byte-for-byte when any write fails — the command must never leave
+ * half the paths pointing at the new root.
+ */
+export type AssetPathStore = {
+  readDocuments(): PersistedAssetPathDocument[];
+  /** Snapshot every file the store can write; returns the backup paths. */
+  backup(): string[];
+  /** Apply all updates as one batch; throws when any write fails. */
+  writeDocuments(updates: AssetPathDocumentUpdate[]): void;
+  /** Put every file named by `backup()` back exactly as it was. */
+  restore(backupPaths: string[]): void;
+};
+
+/** Options for {@link createAppAssetPathStore}. */
+export type AppAssetPathStoreOptions = {
+  /** The auth database holding scheduled messages and drafts; `null` skips the database. */
+  databasePath: string | null;
+  /** Known JSON state files (settings, cockpit card feedback, voice manifest). */
+  jsonFilePaths: string[];
+  /** Clock seam used in backup file names; defaults to the wall clock. */
+  now?: () => Date;
+};
+
+const BACKUP_SUFFIX_MARKER = '.bak-asset-migration-';
+
+/** SQLite columns that can hold an attachment path, grouped per table. */
+const SQLITE_DOCUMENT_COLUMNS: Array<{
+  table: string;
+  keyColumns: string[];
+  columns: string[];
+}> = [
+  { table: 'scheduled_messages', keyColumns: ['id'], columns: ['options'] },
+  {
+    table: 'session_drafts',
+    keyColumns: ['user_id', 'draft_scope'],
+    columns: ['draft_text', 'queued_message'],
+  },
+  {
+    table: 'user_preferences',
+    keyColumns: ['user_id', 'preference_key'],
+    columns: ['preference_value'],
+  },
+];
+
+/** `YYYYMMDD-HHmmss` in local time, used to keep backup names unique per run. */
+function backupStamp(now: () => Date): string {
+  const date = now();
+  const pad = (value: number): string => String(value).padStart(2, '0');
+  return [
+    `${date.getFullYear()}${pad(date.getMonth() + 1)}${pad(date.getDate())}`,
+    `${pad(date.getHours())}${pad(date.getMinutes())}${pad(date.getSeconds())}`,
+  ].join('-');
+}
+
+/** Reads and writes the app-owned path stores; see {@link AssetPathStore}. */
+export function createAppAssetPathStore(options: AppAssetPathStoreOptions): AssetPathStore {
+  const now = options.now ?? (() => new Date());
+  const jsonFilePaths = [...new Set(options.jsonFilePaths)];
+
+  const readableFiles = (): string[] => {
+    const files: string[] = [];
+    if (options.databasePath && fs.existsSync(options.databasePath)) {
+      files.push(options.databasePath);
+    }
+    for (const filePath of jsonFilePaths) {
+      if (fs.existsSync(filePath)) files.push(filePath);
+    }
+    return files;
+  };
+
+  return {
+    readDocuments() {
+      const documents: PersistedAssetPathDocument[] = [];
+      for (const filePath of jsonFilePaths) {
+        if (!fs.existsSync(filePath)) continue;
+        documents.push({
+          location: filePath,
+          text: fs.readFileSync(filePath, 'utf8'),
+          handle: { kind: 'json-file', filePath },
+        });
+      }
+      if (!options.databasePath || !fs.existsSync(options.databasePath)) return documents;
+
+      const database = new Database(options.databasePath, {
+        readonly: true,
+        fileMustExist: true,
+      });
+      try {
+        const tableExists = database.prepare(
+          "SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?",
+        );
+        for (const table of SQLITE_DOCUMENT_COLUMNS) {
+          if (!tableExists.get(table.table)) continue;
+          const selection = [...table.keyColumns, ...table.columns].join(', ');
+          const rows = database
+            .prepare(`SELECT ${selection} FROM ${table.table}`)
+            .all() as Array<Record<string, unknown>>;
+          for (const row of rows) {
+            const keyValues = table.keyColumns.map(
+              (columnName) => row[columnName] as string | number,
+            );
+            for (const column of table.columns) {
+              const value = row[column];
+              if (typeof value !== 'string' || value === '') continue;
+              const keyDescription = table.keyColumns
+                .map((columnName, index) => `${columnName}=${String(keyValues[index])}`)
+                .join(', ');
+              documents.push({
+                location: `${table.table}[${keyDescription}].${column}`,
+                text: value,
+                handle: {
+                  kind: 'sqlite',
+                  table: table.table,
+                  column,
+                  keyColumns: table.keyColumns,
+                  keyValues,
+                },
+              });
+            }
+          }
+        }
+      } finally {
+        database.close();
+      }
+      return documents;
+    },
+
+    backup() {
+      const stamp = backupStamp(now);
+      const backupPaths: string[] = [];
+      for (const filePath of readableFiles()) {
+        const backupPath = `${filePath}${BACKUP_SUFFIX_MARKER}${stamp}`;
+        fs.copyFileSync(filePath, backupPath);
+        backupPaths.push(backupPath);
+      }
+      return backupPaths;
+    },
+
+    writeDocuments(updates) {
+      const jsonUpdates = updates.filter((update) => update.document.handle.kind === 'json-file');
+      const sqliteUpdates = updates.filter((update) => update.document.handle.kind === 'sqlite');
+
+      for (const update of jsonUpdates) {
+        const handle = update.document.handle;
+        if (handle.kind !== 'json-file') continue;
+        fs.writeFileSync(handle.filePath, update.text, 'utf8');
+      }
+
+      if (sqliteUpdates.length === 0) return;
+      if (!options.databasePath) {
+        throw new Error('cannot write SQLite documents without a database path');
+      }
+      const database = new Database(options.databasePath);
+      try {
+        const applyAll = database.transaction(() => {
+          for (const update of sqliteUpdates) {
+            const handle = update.document.handle;
+            if (handle.kind !== 'sqlite') continue;
+            const assignments = `${handle.column} = ?`;
+            const predicates = handle.keyColumns.map((columnName) => `${columnName} = ?`).join(' AND ');
+            database
+              .prepare(`UPDATE ${handle.table} SET ${assignments} WHERE ${predicates}`)
+              .run(update.text, ...handle.keyValues);
+          }
+        });
+        applyAll();
+      } finally {
+        database.close();
+      }
+    },
+
+    restore(backupPaths) {
+      for (const backupPath of backupPaths) {
+        const markerIndex = backupPath.lastIndexOf(BACKUP_SUFFIX_MARKER);
+        if (markerIndex === -1) {
+          throw new Error(`not an asset-migration backup path: ${backupPath}`);
+        }
+        fs.copyFileSync(backupPath, backupPath.slice(0, markerIndex));
+      }
+    },
+  };
+}

package/server/modules/cli/asset-migration.service.ts

@@ -0,0 +1,236 @@
+import path from 'node:path';
+
+import { getImageAssetsRoots } from '@/shared/image-attachments.js';
+import type { CliOutput } from '@/shared/types.js';
+
+import {
+  filesHaveSameContent,
+  listFilesUnder,
+  type AssetMigrationFileSystem,
+} from './asset-migration-files.js';
+import {
+  EXIT_FAILED,
+  EXIT_OK,
+  USAGE,
+  assetMigrationExitCode,
+  parseAssetMigrationOptions,
+  printAssetMigrationDiagnostics,
+  printAssetMigrationReport,
+  type AssetMigrationReport,
+} from './asset-migration-report.js';
+import type { AssetPathDocumentUpdate, AssetPathStore } from './asset-migration-store.js';
+
+/**
+ * `cloudpit migrate-assets` (R4 prerequisite).
+ *
+ * Copies `~/.cloudcli/assets` into the Cloudpit assets root, then rewrites the
+ * legacy absolute paths the app persists itself (the auth database and known
+ * JSON state files). The legacy directory is never deleted or renamed: it stays
+ * as the rollback source and as the read root for references this command
+ * cannot safely rewrite.
+ *
+ * Provider transcripts (Claude JSONL, Codex session JSONL) are external files
+ * owned by the CLIs, so they are scanned read-only and every hit is reported in
+ * `unrewritable`; the command exits non-zero rather than pretending success.
+ */
+
+/** Command surface consumed by `cli.service.ts` (mirrors the sandbox command object). */
+export type AssetMigrationCommandService = {
+  execute(argumentsList: string[]): Promise<number>;
+};
+
+/** Dependencies of {@link createAssetMigrationService}; every seam is injectable. */
+export type AssetMigrationServiceDependencies = {
+  environment: Readonly<Record<string, string | undefined>>;
+  homeDirectory: string;
+  fileSystem: AssetMigrationFileSystem;
+  pathStore: AssetPathStore;
+  output: CliOutput;
+  /** Read-only transcript roots; defaults to the Claude and Codex session trees. */
+  externalRoots?: string[];
+};
+
+/** Provider transcript roots owned by the CLIs; scanned, never rewritten. */
+function defaultExternalRoots(homeDirectory: string): string[] {
+  return [
+    path.join(homeDirectory, '.claude', 'projects'),
+    path.join(homeDirectory, '.codex', 'sessions'),
+  ];
+}
+
+/** Both spellings of a root, so posix-normalized JSON matches on Windows too. */
+function rootVariants(rootPath: string): string[] {
+  return [...new Set([rootPath, rootPath.split(path.sep).join('/')])];
+}
+
+/** Replaces every occurrence of each legacy variant while keeping the form it used. */
+function rewritePaths(
+  text: string,
+  legacyVariants: string[],
+  newRoot: string,
+  legacyRoot: string,
+): string {
+  let rewritten = text;
+  for (const variant of legacyVariants) {
+    const replacement =
+      variant === legacyRoot ? newRoot : newRoot.split(path.sep).join('/');
+    rewritten = rewritten.split(variant).join(replacement);
+  }
+  return rewritten;
+}
+
+/** Collects every external `file:line` hit for the legacy roots, read-only. */
+function collectUnrewritableReferences(
+  fileSystem: AssetMigrationFileSystem,
+  externalRoots: string[],
+  legacyVariants: string[],
+): string[] {
+  const references: string[] = [];
+  for (const rootPath of externalRoots) {
+    for (const filePath of listFilesUnder(fileSystem, rootPath)) {
+      if (!/\.(jsonl|json)$/i.test(filePath)) continue;
+      let text: string;
+      try {
+        text = fileSystem.readTextFile(filePath);
+      } catch {
+        continue;
+      }
+      text.split(/\r?\n/).forEach((line, index) => {
+        if (legacyVariants.some((variant) => line.includes(variant))) {
+          references.push(`${filePath}:${index + 1}`);
+        }
+      });
+    }
+  }
+  return references;
+}
+
+/** One source file and the copy decision made for it. */
+type CopyPlanEntry = {
+  relativePath: string;
+  sourcePath: string;
+  targetPath: string;
+  action: 'copy' | 'skip' | 'conflict';
+};
+
+/**
+ * Creates the migrate-assets command service.
+ *
+ * Copies are additive and idempotent, so a partial copy is safe; the path
+ * rewrite is the destructive half, which is why it alone takes backups and
+ * rolls the whole batch back on failure.
+ */
+export function createAssetMigrationService(
+  dependencies: AssetMigrationServiceDependencies,
+): AssetMigrationCommandService {
+  const { fileSystem, output } = dependencies;
+
+  return {
+    async execute(argumentsList) {
+      if (argumentsList.includes('--help') || argumentsList.includes('-h')) {
+        output.log(USAGE);
+        return EXIT_OK;
+      }
+      if (argumentsList.length === 0) {
+        output.log(USAGE);
+        return EXIT_FAILED;
+      }
+
+      const parsed = parseAssetMigrationOptions(argumentsList);
+      if (typeof parsed === 'string') {
+        output.error(`[ERROR] ${parsed}`);
+        output.log(USAGE);
+        return EXIT_FAILED;
+      }
+
+      const roots = getImageAssetsRoots({
+        env: dependencies.environment,
+        homeDir: dependencies.homeDirectory,
+      });
+      const legacyRoot = roots.legacyRoot;
+      const newRoot = roots.newRoot;
+      const legacyVariants = rootVariants(legacyRoot);
+      const report: AssetMigrationReport = {
+        copied: 0,
+        skipped: 0,
+        conflicts: [],
+        rewritten: 0,
+        unrewritable: [],
+        backupPaths: [],
+        errors: [],
+      };
+
+      const copyPlan: CopyPlanEntry[] = [];
+      for (const sourcePath of listFilesUnder(fileSystem, legacyRoot)) {
+        const relativePath = path.relative(legacyRoot, sourcePath);
+        const targetPath = path.join(newRoot, relativePath);
+        if (!fileSystem.pathExists(targetPath)) {
+          copyPlan.push({ relativePath, sourcePath, targetPath, action: 'copy' });
+          report.copied += 1;
+        } else if (filesHaveSameContent(fileSystem, sourcePath, targetPath)) {
+          copyPlan.push({ relativePath, sourcePath, targetPath, action: 'skip' });
+          report.skipped += 1;
+        } else {
+          copyPlan.push({ relativePath, sourcePath, targetPath, action: 'conflict' });
+          report.conflicts.push(relativePath);
+        }
+      }
+
+      const updates: AssetPathDocumentUpdate[] = [];
+      for (const document of dependencies.pathStore.readDocuments()) {
+        if (!legacyVariants.some((variant) => document.text.includes(variant))) continue;
+        updates.push({
+          document,
+          text: rewritePaths(document.text, legacyVariants, newRoot, legacyRoot),
+        });
+      }
+      report.rewritten = updates.length;
+
+      report.unrewritable = collectUnrewritableReferences(
+        fileSystem,
+        dependencies.externalRoots ?? defaultExternalRoots(dependencies.homeDirectory),
+        legacyVariants,
+      );
+
+      if (parsed.mode === 'rewrite') {
+        for (const entry of copyPlan) {
+          if (entry.action !== 'copy') continue;
+          try {
+            fileSystem.createDirectory(path.dirname(entry.targetPath));
+            fileSystem.copyFile(entry.sourcePath, entry.targetPath);
+          } catch (error) {
+            report.errors.push(`copy ${entry.relativePath}: ${describeFailure(error)}`);
+          }
+        }
+
+        if (updates.length > 0) {
+          report.backupPaths = dependencies.pathStore.backup();
+          try {
+            dependencies.pathStore.writeDocuments(updates);
+          } catch (error) {
+            report.errors.push(`rewrite: ${describeFailure(error)}`);
+            report.rewritten = 0;
+            try {
+              dependencies.pathStore.restore(report.backupPaths);
+            } catch (restoreError) {
+              report.errors.push(`restore: ${describeFailure(restoreError)}`);
+            }
+          }
+        }
+      }
+
+      if (parsed.json) {
+        output.log(JSON.stringify(report, null, 2));
+      } else {
+        printAssetMigrationReport(output, report, parsed.mode, legacyRoot, newRoot);
+      }
+      printAssetMigrationDiagnostics(output, report, legacyRoot);
+      return assetMigrationExitCode(report);
+    },
+  };
+}
+
+/** Error message without importing a shared helper name the architecture ratchet counts. */
+function describeFailure(failure: unknown): string {
+  return failure instanceof Error ? failure.message : String(failure);
+}

package/server/modules/cli/browser-use-mcp.ts

@@ -9,6 +9,7 @@
 // The MCP executable must load the root environment bootstrap before reading configuration.
 // eslint-disable-next-line boundaries/no-unknown
 import '../../load-env.js';
+import { createBrandEnvReader } from '../../shared/brand.js';
 
 type JsonRpcRequest = {
   jsonrpc: '2.0';
@@ -42,13 +43,17 @@ const readOptionalString = (value: unknown): string | undefined =>
 const readNumber = (value: unknown): number | undefined =>
   typeof value === 'number' && Number.isFinite(value) ? value : undefined;
 
-const apiUrl = (process.env.CLOUDCLI_BROWSER_USE_API_URL || 'http://127.0.0.1:3001/api/browser-use-mcp').replace(/\/$/, '');
-const apiToken = process.env.CLOUDCLI_BROWSER_USE_MCP_TOKEN || '';
-const API_TIMEOUT_MS = Number.parseInt(process.env.CLOUDCLI_BROWSER_USE_API_TIMEOUT_MS || '60000', 10);
+// The environment bootstrap above has already loaded `.env`; the reader is
+// built here so the canonical `CLOUDPIT_*` names resolve against the final
+// process env. The legacy `CLOUDCLI_*` spellings are no longer read (v3.0).
+const brandEnv = createBrandEnvReader({ env: process.env });
+const apiUrl = (brandEnv.read('BROWSER_USE_API_URL').value || 'http://127.0.0.1:3001/api/browser-use-mcp').replace(/\/$/, '');
+const apiToken = brandEnv.read('BROWSER_USE_MCP_TOKEN').value || '';
+const API_TIMEOUT_MS = Number.parseInt(brandEnv.read('BROWSER_USE_API_TIMEOUT_MS').value || '60000', 10);
 
 async function callBrowserUseApi(toolName: string, input: Record<string, unknown>) {
   if (!apiToken) {
-    throw new Error('CLOUDCLI_BROWSER_USE_MCP_TOKEN is not configured.');
+    throw new Error('CLOUDPIT_BROWSER_USE_MCP_TOKEN is not configured.');
   }
 
   const response = await fetch(`${apiUrl}/tools/${encodeURIComponent(toolName)}`, {

package/server/modules/cli/cli.module.ts

@@ -10,6 +10,9 @@ import { findApplicationRoot, getModuleDirectory } from '@/shared/utils.js';
 import { resolveDataRoots } from '@/shared/data-root.js';
 
 import { createCliService } from './cli.service.js';
+import { createNodeAssetMigrationFileSystem } from './asset-migration-files.js';
+import { createAssetMigrationService } from './asset-migration.service.js';
+import { createAppAssetPathStore } from './asset-migration-store.js';
 import { createSandboxCommandService } from './sandbox.service.js';
 
 /**
@@ -40,6 +43,7 @@ export function createCliApplication(): CliApplication {
     error: (message?: string) => console.error(message),
   };
   const homeDirectory = os.homedir();
+  const dataRoots = resolveDataRoots({ homeDir: homeDirectory });
   const sandboxService = createSandboxCommandService({
     homeDirectory,
     fileSystem,
@@ -63,13 +67,29 @@ export function createCliApplication(): CliApplication {
 
   return createCliService({
     applicationRoot,
-    defaultDatabasePath: resolveDataRoots({ defaultRoot: 'cloudpit', homeDir: homeDirectory }).authDatabasePath,
+    defaultDatabasePath: dataRoots.authDatabasePath,
     homeDirectory,
     packageMetadata,
     environment: process.env,
     fileSystem,
     output,
     sandboxService,
+    assetMigrationService: createAssetMigrationService({
+      environment: process.env,
+      homeDirectory,
+      fileSystem: createNodeAssetMigrationFileSystem(),
+      pathStore: createAppAssetPathStore({
+        // The server may have been started with an explicit database; the command
+        // must rewrite that same store, not the default one.
+        databasePath: process.env.DATABASE_PATH || dataRoots.authDatabasePath,
+        jsonFilePaths: [
+          dataRoots.settingsPath,
+          dataRoots.cockpitCardFeedbackPath,
+          path.join(dataRoots.voiceDataDir, 'manifest.json'),
+        ],
+      }),
+      output,
+    }),
     getLatestPackageVersion: async () => {
       // Yield first so the default `start` command can begin loading the server
       // before this best-effort npm registry check runs.

package/server/modules/cli/cli.service.ts

@@ -19,6 +19,14 @@ type CliServiceDependencies = {
   fileSystem: CliFileSystem;
   output: CliOutput;
   sandboxService: SandboxCommandService;
+  /**
+   * The `migrate-assets` subcommand. It is a separate command object like the
+   * sandbox service so tests can stub the filesystem-heavy migration instead of
+   * running it.
+   */
+  assetMigrationService: {
+    execute(argumentsList: string[]): Promise<number>;
+  };
   getLatestPackageVersion(): Promise<string>;
   updateGlobalPackage(): void;
   startServer(): Promise<void>;
@@ -57,7 +65,9 @@ function parseCliArguments(argumentsList: string[]): ParsedCliArguments {
       parsedArguments.command = 'version';
     } else if (!argument.startsWith('-')) {
       parsedArguments.command = argument;
-      if (argument === 'sandbox') {
+      // Subcommands that own their remaining arguments (flags such as
+      // `--dry-run` or `-p` belong to them, not to the top-level parser).
+      if (argument === 'sandbox' || argument === 'migrate-assets') {
         parsedArguments.remainingArguments = argumentsList.slice(argumentIndex + 1);
         break;
       }
@@ -137,6 +147,7 @@ Usage:
 Commands:
   start            Start the Cloudpit server (default)
   sandbox          Manage Docker sandbox environments
+  migrate-assets   Copy legacy ~/.cloudcli/assets into the Cloudpit root
   browser-use-mcp  Run Browser MCP stdio server
   status           Show configuration and data locations
   update           Update to the latest version
@@ -235,6 +246,8 @@ export function createCliService(dependencies: CliServiceDependencies): CliAppli
           return 0;
         case 'sandbox':
           return dependencies.sandboxService.execute(parsedArguments.remainingArguments);
+        case 'migrate-assets':
+          return dependencies.assetMigrationService.execute(parsedArguments.remainingArguments);
         case 'browser-use-mcp':
           await dependencies.startBrowserUseMcp();
           return 0;