ic-mops 2.10.0 → 2.12.0

package/CHANGELOG.md

@@ -2,6 +2,17 @@
 
 ## Next
 
+## 2.12.0
+- Migration staging directory moved from `.mops/.migrations/<canister>/` to `<parent-of-chain>/.migrations-<canister>/`, so migration files can import shared modules from sibling folders (e.g. a `types/` folder next to `migrations/`) — relative imports now resolve to the same target whether moc reads the original chain dir or the staged one. The staged dir self-stamps a `.gitignore` so it doesn't pollute `git status`; `mops init` now also adds `.migrations-*/` to the project `.gitignore`
+- `[canisters.<name>.migrations]` now requires `chain` and `next` to share the same parent directory (any layout where the parents differed is rejected with a clear error). The default layout `chain = "migrations"` + `next = "next-migration"` already satisfies this. For per-canister setups, use sibling subdirectories, e.g. `chain = "src/backend/migrations"` + `next = "src/backend/next-migration"`
+
+## 2.11.0
+- Add `mops migrate new <Name>` and `mops migrate freeze` commands for managing enhanced migration chains
+- Add `[canisters.<name>.migrations]` config section with `chain`, `next`, `check-limit`, and `build-limit` fields
+- `mops check`, `mops build`, and `mops check-stable` now auto-inject `--enhanced-migration` when `[migrations]` is configured
+- `mops check` and `mops check-stable` emit a hint to create a migration when a stable compatibility check fails and `[migrations]` is configured
+- Migration chain trimming: only the last N migrations are passed to `moc` based on `check-limit`/`build-limit` settings
+
 ## 2.10.0
 - `mops check` and `mops check-stable` now apply per-canister `[canisters.<name>].args` (previously only `mops build` applied them)
 - `mops check` now accepts canister names as arguments (e.g. `mops check backend`) to check a specific canister

package/cli.ts

@@ -43,6 +43,7 @@ import {
   importPem,
   setUserProp,
 } from "./commands/user.js";
+import { migrateNew, migrateFreeze } from "./commands/migrate.js";
 import { watch } from "./commands/watch/watch.js";
 import {
   apiVersion,
@@ -707,6 +708,29 @@ toolchainCommand
 
 program.addCommand(toolchainCommand);
 
+// migrate
+const migrateCommand = new Command("migrate").description(
+  "Manage enhanced migration chains",
+);
+
+migrateCommand
+  .command("new <name> [canister]")
+  .description("Create a new migration file in the next-migration directory")
+  .action(async (name, canister) => {
+    checkConfigFile(true);
+    await migrateNew(name, canister);
+  });
+
+migrateCommand
+  .command("freeze [canister]")
+  .description("Move the next migration into the frozen chain")
+  .action(async (canister) => {
+    checkConfigFile(true);
+    await migrateFreeze(canister);
+  });
+
+program.addCommand(migrateCommand);
+
 // self
 const selfCommand = new Command("self").description("Mops CLI management");
 

package/commands/build.ts

@@ -11,6 +11,7 @@ import {
   resolveCanisterConfigs,
   validateCanisterArgs,
 } from "../helpers/resolve-canisters.js";
+import { prepareMigrationArgs } from "../helpers/migrations.js";
 import { CanisterConfig, Config } from "../types.js";
 import { CustomSection, getWasmBindings } from "../wasm.js";
 import { getGlobalMocArgs, readConfig, resolveConfigPath } from "../mops.js";
@@ -87,6 +88,12 @@ export async function build(
     };
     process.on("exit", exitCleanup);
 
+    const migration = await prepareMigrationArgs(
+      canister.migrations,
+      canisterName,
+      "build",
+      options.verbose,
+    );
     try {
       let args = [
         "-c",
@@ -97,6 +104,7 @@ export async function build(
         motokoPath,
         ...(await sourcesArgs()).flat(),
         ...getGlobalMocArgs(config),
+        ...migration.migrationArgs,
       ];
       args.push(
         ...collectExtraArgs(config, canister, canisterName, options.extraArgs),
@@ -199,6 +207,7 @@ export async function build(
         );
       }
     } finally {
+      await migration.cleanup();
       process.removeListener("exit", exitCleanup);
       try {
         await release?.();
@@ -238,7 +247,7 @@ function collectExtraArgs(
     args.push(...config.build.args);
   }
   if (canister.args) {
-    validateCanisterArgs(canister, canisterName);
+    validateCanisterArgs(canister, canisterName, config);
     args.push(...canister.args);
   }
   if (extraArgs) {

package/commands/check-stable.ts

@@ -4,6 +4,7 @@ import { rm } from "node:fs/promises";
 import chalk from "chalk";
 import { execa } from "execa";
 import { cliError } from "../error.js";
+import { prepareMigrationArgs } from "../helpers/migrations.js";
 import { getGlobalMocArgs, readConfig, resolveConfigPath } from "../mops.js";
 import { CanisterConfig } from "../types.js";
 import {
@@ -70,17 +71,28 @@ export async function checkStable(
       cliError(`No main file specified for canister '${name}' in mops.toml`);
     }
 
-    validateCanisterArgs(canister, name);
+    validateCanisterArgs(canister, name, config);
 
-    await runStableCheck({
-      oldFile,
-      canisterMain: resolveConfigPath(canister.main),
-      canisterName: name,
-      mocPath,
-      globalMocArgs,
-      canisterArgs: canister.args ?? [],
-      options,
-    });
+    const migration = await prepareMigrationArgs(
+      canister.migrations,
+      name,
+      "check",
+      options.verbose,
+    );
+    try {
+      await runStableCheck({
+        oldFile,
+        canisterMain: resolveConfigPath(canister.main),
+        canisterName: name,
+        mocPath,
+        globalMocArgs,
+        canisterArgs: [...migration.migrationArgs, ...(canister.args ?? [])],
+        options,
+        hasMigrations: !!canister.migrations,
+      });
+    } finally {
+      await migration.cleanup();
+    }
     return;
   }
 
@@ -95,7 +107,7 @@ export async function checkStable(
       cliError(`No main file specified for canister '${name}' in mops.toml`);
     }
 
-    validateCanisterArgs(canister, name);
+    validateCanisterArgs(canister, name, config);
     const stablePath = resolveStablePath(canister, name, {
       required: !!canisterNames,
     });
@@ -103,16 +115,27 @@ export async function checkStable(
       continue;
     }
 
-    await runStableCheck({
-      oldFile: stablePath,
-      canisterMain: resolveConfigPath(canister.main),
-      canisterName: name,
-      mocPath,
-      globalMocArgs,
-      canisterArgs: canister.args ?? [],
-      sources,
-      options,
-    });
+    const migration = await prepareMigrationArgs(
+      canister.migrations,
+      name,
+      "check",
+      options.verbose,
+    );
+    try {
+      await runStableCheck({
+        oldFile: stablePath,
+        canisterMain: resolveConfigPath(canister.main),
+        canisterName: name,
+        mocPath,
+        globalMocArgs,
+        canisterArgs: [...migration.migrationArgs, ...(canister.args ?? [])],
+        sources,
+        options,
+        hasMigrations: !!canister.migrations,
+      });
+    } finally {
+      await migration.cleanup();
+    }
     checked++;
   }
 
@@ -136,6 +159,7 @@ export interface RunStableCheckParams {
   canisterArgs: string[];
   sources?: string[];
   options?: Partial<CheckStableOptions>;
+  hasMigrations?: boolean;
 }
 
 export async function runStableCheck(
@@ -204,6 +228,13 @@ export async function runStableCheck(
       if (result.stderr) {
         console.error(result.stderr);
       }
+      if (params.hasMigrations) {
+        console.error(
+          chalk.yellow(
+            "Hint: You may need a migration. Run `mops migrate new <Name>` to create one.",
+          ),
+        );
+      }
       cliError(
         `✗ Stable compatibility check failed for canister '${canisterName}'`,
       );

package/commands/check.ts

@@ -16,6 +16,7 @@ import {
   resolveCanisterConfigs,
   validateCanisterArgs,
 } from "../helpers/resolve-canisters.js";
+import { prepareMigrationArgs } from "../helpers/migrations.js";
 import { CanisterConfig, Config } from "../types.js";
 import { resolveStablePath, runStableCheck } from "./check-stable.js";
 import { sourcesArgs } from "./sources.js";
@@ -144,70 +145,82 @@ async function checkCanisters(
       );
     }
 
-    validateCanisterArgs(canister, canisterName);
+    validateCanisterArgs(canister, canisterName, config);
     const motokoPath = resolveConfigPath(canister.main);
 
-    const mocArgs = [
-      "--check",
-      ...(allLibs ? ["--all-libs"] : []),
-      ...sources,
-      ...globalMocArgs,
-      ...(canister.args ?? []),
-      ...(options.extraArgs ?? []),
-    ];
+    const migration = await prepareMigrationArgs(
+      canister.migrations,
+      canisterName,
+      "check",
+      options.verbose,
+    );
+    try {
+      const mocArgs = [
+        "--check",
+        ...(allLibs ? ["--all-libs"] : []),
+        ...sources,
+        ...globalMocArgs,
+        ...migration.migrationArgs,
+        ...(canister.args ?? []),
+        ...(options.extraArgs ?? []),
+      ];
 
-    if (options.fix) {
-      if (options.verbose) {
-        console.log(
-          chalk.blue("check"),
-          chalk.gray(`Attempting to fix ${canisterName}`),
-        );
+      if (options.fix) {
+        if (options.verbose) {
+          console.log(
+            chalk.blue("check"),
+            chalk.gray(`Attempting to fix ${canisterName}`),
+          );
+        }
+
+        const fixResult = await autofixMotoko(mocPath, [motokoPath], mocArgs);
+        logAutofixResult(fixResult, options.verbose);
       }
 
-      const fixResult = await autofixMotoko(mocPath, [motokoPath], mocArgs);
-      logAutofixResult(fixResult, options.verbose);
-    }
+      try {
+        const args = [motokoPath, ...mocArgs];
+        if (options.verbose) {
+          console.log(
+            chalk.blue("check"),
+            chalk.gray(`Checking canister ${canisterName}:`),
+          );
+          console.log(chalk.gray(mocPath, JSON.stringify(args)));
+        }
 
-    try {
-      const args = [motokoPath, ...mocArgs];
-      if (options.verbose) {
-        console.log(
-          chalk.blue("check"),
-          chalk.gray(`Checking canister ${canisterName}:`),
-        );
-        console.log(chalk.gray(mocPath, JSON.stringify(args)));
-      }
+        const result = await execa(mocPath, args, {
+          stdio: "inherit",
+          reject: false,
+        });
 
-      const result = await execa(mocPath, args, {
-        stdio: "inherit",
-        reject: false,
-      });
+        if (result.exitCode !== 0) {
+          cliError(
+            `✗ Check failed for canister ${canisterName} (exit code: ${result.exitCode})`,
+          );
+        }
 
-      if (result.exitCode !== 0) {
+        console.log(chalk.green(`✓ ${canisterName}`));
+      } catch (err: any) {
         cliError(
-          `✗ Check failed for canister ${canisterName} (exit code: ${result.exitCode})`,
+          `Error while checking canister ${canisterName}${err?.message ? `\n${err.message}` : ""}`,
         );
       }
 
-      console.log(chalk.green(`✓ ${canisterName}`));
-    } catch (err: any) {
-      cliError(
-        `Error while checking canister ${canisterName}${err?.message ? `\n${err.message}` : ""}`,
-      );
-    }
-
-    const stablePath = resolveStablePath(canister, canisterName);
-    if (stablePath) {
-      await runStableCheck({
-        oldFile: stablePath,
-        canisterMain: motokoPath,
-        canisterName,
-        mocPath,
-        globalMocArgs,
-        canisterArgs: canister.args ?? [],
-        sources,
-        options: { verbose: options.verbose, extraArgs: options.extraArgs },
-      });
+      const stablePath = resolveStablePath(canister, canisterName);
+      if (stablePath) {
+        await runStableCheck({
+          oldFile: stablePath,
+          canisterMain: motokoPath,
+          canisterName,
+          mocPath,
+          globalMocArgs,
+          canisterArgs: [...migration.migrationArgs, ...(canister.args ?? [])],
+          sources,
+          options: { verbose: options.verbose, extraArgs: options.extraArgs },
+          hasMigrations: !!canister.migrations,
+        });
+      }
+    } finally {
+      await migration.cleanup();
     }
   }
 }

package/commands/init.ts

@@ -282,16 +282,29 @@ async function applyInit({
     await template("github-workflow:mops-test");
   }
 
-  // add .mops to .gitignore
+  // add mops-managed paths to .gitignore
   {
     let gitignore = path.join(process.cwd(), ".gitignore");
     let gitignoreData = existsSync(gitignore)
       ? readFileSync(gitignore).toString()
       : "";
-    let lf = gitignoreData.endsWith("\n") ? "\n" : "";
+    const additions: string[] = [];
     if (!gitignoreData.includes(".mops")) {
-      writeFileSync(gitignore, `${gitignoreData}\n.mops${lf}`.trimStart());
-      console.log(chalk.green("Added"), ".mops to .gitignore");
+      additions.push(".mops");
+    }
+    if (!gitignoreData.includes(".migrations-")) {
+      additions.push(".migrations-*/");
+    }
+    if (additions.length > 0) {
+      let lf = gitignoreData.endsWith("\n") ? "\n" : "";
+      writeFileSync(
+        gitignore,
+        `${gitignoreData}\n${additions.join("\n")}${lf}`.trimStart(),
+      );
+      console.log(
+        chalk.green("Added"),
+        `${additions.join(", ")} to .gitignore`,
+      );
     }
   }
 

package/commands/migrate.ts

@@ -0,0 +1,165 @@
+import { existsSync, mkdirSync, renameSync } from "node:fs";
+import { writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import chalk from "chalk";
+import { cliError } from "../error.js";
+import {
+  getNextMigrationFile,
+  validateMigrationsConfig,
+  validateNextMigrationOrder,
+} from "../helpers/migrations.js";
+import { resolveCanisterConfigs } from "../helpers/resolve-canisters.js";
+import { readConfig, resolveConfigPath } from "../mops.js";
+import { CanisterConfig } from "../types.js";
+
+function resolveMigrationCanister(canisterName?: string): {
+  name: string;
+  canister: CanisterConfig;
+} {
+  const config = readConfig();
+  const canisters = resolveCanisterConfigs(config);
+  const withMigrations = Object.entries(canisters).filter(
+    ([, c]) => c.migrations,
+  );
+
+  if (withMigrations.length === 0) {
+    cliError(
+      "No canisters with [migrations] config found in mops.toml.\n" +
+        "Add a [canisters.<name>.migrations] section first:\n\n" +
+        "  [canisters.backend.migrations]\n" +
+        '  chain = "migrations"\n' +
+        '  next = "next-migration"   # required for migrate new/freeze',
+    );
+  }
+
+  if (canisterName) {
+    const canister = canisters[canisterName];
+    if (!canister) {
+      cliError(
+        `Canister '${canisterName}' not found in mops.toml. Available: ${Object.keys(canisters).join(", ")}`,
+      );
+    }
+    if (!canister.migrations) {
+      cliError(
+        `Canister '${canisterName}' has no [canisters.${canisterName}.migrations] config in mops.toml`,
+      );
+    }
+    return { name: canisterName, canister };
+  }
+
+  if (withMigrations.length > 1) {
+    cliError(
+      `Multiple canisters with [migrations] config. Please specify one: ${withMigrations.map(([n]) => n).join(", ")}`,
+    );
+  }
+
+  return { name: withMigrations[0]![0], canister: withMigrations[0]![1] };
+}
+
+const VALID_NAME_RE = /^[A-Za-z][A-Za-z0-9_]*$/;
+
+function generateTimestamp(): string {
+  const now = new Date();
+  const pad = (n: number) => String(n).padStart(2, "0");
+  return (
+    `${now.getUTCFullYear()}${pad(now.getUTCMonth() + 1)}${pad(now.getUTCDate())}` +
+    `_${pad(now.getUTCHours())}${pad(now.getUTCMinutes())}${pad(now.getUTCSeconds())}`
+  );
+}
+
+const MIGRATION_TEMPLATE = `module {
+  public func migration(old : {}) : {} {
+    {}
+  }
+}
+`;
+
+export async function migrateNew(
+  name: string,
+  canisterName?: string,
+): Promise<void> {
+  if (!VALID_NAME_RE.test(name)) {
+    cliError(
+      `Invalid migration name: "${name}"\n` +
+        "Name must start with a letter and contain only letters, digits, and underscores.",
+    );
+  }
+
+  const { name: resolvedName, canister } =
+    resolveMigrationCanister(canisterName);
+  const migrations = canister.migrations!;
+  validateMigrationsConfig(migrations, resolvedName);
+
+  if (!migrations.next) {
+    cliError(
+      `[canisters.${resolvedName}.migrations] is missing the "next" field.\n` +
+        'Add next = "next-migration" to use `mops migrate new/freeze`.',
+    );
+  }
+
+  const chainDir = resolveConfigPath(migrations.chain);
+  const nextDir = resolveConfigPath(migrations.next);
+
+  const existingNext = existsSync(nextDir)
+    ? getNextMigrationFile(nextDir)
+    : null;
+  if (existingNext) {
+    cliError(
+      `A next migration already exists: ${existingNext}\n` +
+        "Freeze it first with `mops migrate freeze`.",
+    );
+  }
+
+  const timestamp = generateTimestamp();
+  const fileName = `${timestamp}_${name}.mo`;
+
+  validateNextMigrationOrder(chainDir, fileName);
+
+  if (!existsSync(chainDir)) {
+    mkdirSync(chainDir, { recursive: true });
+  }
+  if (!existsSync(nextDir)) {
+    mkdirSync(nextDir, { recursive: true });
+  }
+
+  const filePath = join(nextDir, fileName);
+  await writeFile(filePath, MIGRATION_TEMPLATE);
+
+  console.log(chalk.green(`✓ Created migration: ${filePath}`));
+}
+
+export async function migrateFreeze(canisterName?: string): Promise<void> {
+  const { name: resolvedName, canister } =
+    resolveMigrationCanister(canisterName);
+  const migrations = canister.migrations!;
+  validateMigrationsConfig(migrations, resolvedName);
+
+  if (!migrations.next) {
+    cliError(
+      `[canisters.${resolvedName}.migrations] is missing the "next" field.\n` +
+        'Add next = "next-migration" to use `mops migrate new/freeze`.',
+    );
+  }
+
+  const chainDir = resolveConfigPath(migrations.chain);
+  const nextDir = resolveConfigPath(migrations.next);
+
+  const nextFile = existsSync(nextDir) ? getNextMigrationFile(nextDir) : null;
+  if (!nextFile) {
+    cliError(
+      "No next migration to freeze. Create one with `mops migrate new <Name>`.",
+    );
+  }
+
+  validateNextMigrationOrder(chainDir, nextFile);
+
+  if (!existsSync(chainDir)) {
+    mkdirSync(chainDir, { recursive: true });
+  }
+
+  const src = join(nextDir, nextFile);
+  const dest = join(chainDir, nextFile);
+  renameSync(src, dest);
+
+  console.log(chalk.green(`✓ Frozen migration: ${nextFile} → ${chainDir}/`));
+}

package/declarations/main/main.did

@@ -42,6 +42,11 @@ type StreamingCallbackResponse =
  };
 type StreamingCallback = func (StreamingToken) ->
                           (opt StreamingCallbackResponse) query;
+type StructureStats = 
+ record {
+   bytes: nat;
+   count: nat;
+ };
 type StorageStats = 
  record {
    cyclesBalance: nat;
@@ -272,6 +277,38 @@ type PackageChanges =
    prevDocsCoverage: float64;
    tests: TestsChanges;
  };
+type MemoryStats = 
+ record {
+   dailySnapshots: StructureStats;
+   dailySnapshotsByPackageId: StructureStats;
+   dailySnapshotsByPackageName: StructureStats;
+   dailyTempRecords: StructureStats;
+   downloadsByPackageId: StructureStats;
+   downloadsByPackageName: StructureStats;
+   fileIdsByPackage: StructureStats;
+   hashByFileId: StructureStats;
+   highestConfigs: StructureStats;
+   maintainersByPackage: StructureStats;
+   ownersByPackage: StructureStats;
+   packageBenchmarks: StructureStats;
+   packageConfigs: StructureStats;
+   packageDocsCoverage: StructureStats;
+   packageFileStats: StructureStats;
+   packageNotes: StructureStats;
+   packagePublications: StructureStats;
+   packageTestStats: StructureStats;
+   packageVersions: StructureStats;
+   rtsHeapSize: nat;
+   rtsMemorySize: nat;
+   names: StructureStats;
+   storageByFileId: StructureStats;
+   storages: StructureStats;
+   users: StructureStats;
+   weeklySnapshots: StructureStats;
+   weeklySnapshotsByPackageId: StructureStats;
+   weeklySnapshotsByPackageName: StructureStats;
+   weeklyTempRecords: StructureStats;
+ };
 type Main = 
  service {
    addMaintainer: (packageName: PackageName, newMaintainer: principal) ->
@@ -311,6 +348,7 @@ type Main =
           SemverPart;
         }) -> (Result_6) query;
    getHighestVersion: (name: PackageName) -> (Result_5) query;
+   getMemoryStats: () -> (MemoryStats) query;
    getMostDownloadedPackages: () -> (vec PackageSummary) query;
    getMostDownloadedPackagesIn7Days: () -> (vec PackageSummary) query;
    getNewPackages: () -> (vec PackageSummary) query;

package/declarations/main/main.did.d.ts

@@ -45,6 +45,41 @@ export interface HttpRequestResult {
   'body' : Uint8Array | number[],
   'headers' : Array<HttpHeader>,
 }
+export interface StructureStats {
+  'count' : bigint,
+  'bytes' : bigint,
+}
+export interface MemoryStats {
+  'rtsHeapSize' : bigint,
+  'rtsMemorySize' : bigint,
+  'packageVersions' : StructureStats,
+  'packageConfigs' : StructureStats,
+  'highestConfigs' : StructureStats,
+  'packagePublications' : StructureStats,
+  'ownersByPackage' : StructureStats,
+  'maintainersByPackage' : StructureStats,
+  'fileIdsByPackage' : StructureStats,
+  'hashByFileId' : StructureStats,
+  'packageFileStats' : StructureStats,
+  'packageTestStats' : StructureStats,
+  'packageBenchmarks' : StructureStats,
+  'packageNotes' : StructureStats,
+  'packageDocsCoverage' : StructureStats,
+  'downloadsByPackageName' : StructureStats,
+  'downloadsByPackageId' : StructureStats,
+  'dailySnapshots' : StructureStats,
+  'weeklySnapshots' : StructureStats,
+  'dailySnapshotsByPackageName' : StructureStats,
+  'dailySnapshotsByPackageId' : StructureStats,
+  'weeklySnapshotsByPackageName' : StructureStats,
+  'weeklySnapshotsByPackageId' : StructureStats,
+  'dailyTempRecords' : StructureStats,
+  'weeklyTempRecords' : StructureStats,
+  'storages' : StructureStats,
+  'storageByFileId' : StructureStats,
+  'users' : StructureStats,
+  'names' : StructureStats,
+}
 export interface Main {
   'addMaintainer' : ActorMethod<[PackageName, Principal], Result_3>,
   'addOwner' : ActorMethod<[PackageName, Principal], Result_3>,
@@ -77,6 +112,7 @@ export interface Main {
     Result_6
   >,
   'getHighestVersion' : ActorMethod<[PackageName], Result_5>,
+  'getMemoryStats' : ActorMethod<[], MemoryStats, 'query'>,
   'getMostDownloadedPackages' : ActorMethod<[], Array<PackageSummary>>,
   'getMostDownloadedPackagesIn7Days' : ActorMethod<[], Array<PackageSummary>>,
   'getNewPackages' : ActorMethod<[], Array<PackageSummary>>,