gitmem-mcp 1.6.0 → 1.6.1

package/CHANGELOG.md

@@ -7,6 +7,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [1.6.1] - 2026-05-25
+
+### Fixed
+- **Embeddings not generated on Pro tier**: `embedding.ts`, `variant-generation.ts`, and `transcript-chunker.ts` only checked `process.env.OPENROUTER_API_KEY` — never read the key from `.gitmem/config.json` written by `activate`. Now falls back to `getProConfig()` matching how `supabase-client.ts` already resolves credentials.
+- **Lossy free→pro migration**: Migration sent all local JSON fields to PostgREST — unknown columns caused 400 rejections, silently dropping records (only first 3 errors shown). Added `KNOWN_COLUMNS` whitelist per table, type coercion for `action_protocol`/`self_check_criteria` (array→TEXT), and full error visibility (no cap).
+- **Migration log file**: `.gitmem/migration.log` now written with per-record outcomes (OK/FAIL/SKIP) for debugging.
+- **Mid-migration recovery**: `activate` now detects `.pre-migration` backup files from a previous failed upgrade and re-imports them automatically. No new command — just re-run `activate`.
+- **Credential exposure**: `activate` now auto-adds `.gitmem/` to the project's `.gitignore` when inside a git repo.
+
+### Changed
+- **E2E stress test v1.4**: Restructured from 6 to 8 simulated days. Day 0 wipes Supabase to blank slate. Day 1 seeds realistic 3+ month free-tier user data (starter scars, unknown fields, type mismatches, mixed projects) and tests full upgrade journey including mid-failure recovery and real embeddings from config.json (not env var). 178 tests total.
+
 ## [1.6.0] - 2026-05-25
 
 ### Added

package/dist/commands/activate.js

@@ -22,7 +22,7 @@ import * as readline from "readline";
 import { fileURLToPath } from "url";
 import { getGitmemDir, getInstallId } from "../services/gitmem-dir.js";
 import { validateLicense, clearLicenseCache } from "../services/license.js";
-import { hasLocalData, migrateLocalToSupabase, archiveLocalData } from "./migrate-local.js";
+import { hasLocalData, hasPreMigrationData, migrateLocalToSupabase, reimportFromBackups, archiveLocalData } from "./migrate-local.js";
 function createReadline() {
     return readline.createInterface({
         input: process.stdin,
@@ -485,56 +485,100 @@ export async function main(args) {
     fs.writeFileSync(configPath, JSON.stringify(config, null, 2));
     // Clear any stale license cache
     clearLicenseCache();
-    // Step 7: Migrate local data to Supabase (free → pro upgrade)
-    if (supabaseUrl && supabaseKey && missingTables.length === 0 && hasLocalData(gitmemDir)) {
-        console.log("Migrating Local Data");
-        console.log("  Found existing local data from free tier...");
-        console.log("");
-        const migrationResult = await migrateLocalToSupabase({
-            supabaseUrl,
-            supabaseKey,
-            gitmemDir,
-            onProgress: (msg) => console.log(msg),
-        });
-        // Report results
-        const collections = Object.keys(migrationResult.migrated);
-        let totalMigrated = 0;
-        let totalSkipped = 0;
-        let totalErrors = 0;
-        for (const col of collections) {
-            const m = migrationResult.migrated[col];
-            const s = migrationResult.skipped[col];
-            const e = migrationResult.errors[col]?.length || 0;
-            totalMigrated += m;
-            totalSkipped += s;
-            totalErrors += e;
-            if (m > 0) {
-                console.log(`  ✓ ${col}: ${m} records migrated${s > 0 ? ` (${s} skipped)` : ""}`);
+    // Ensure .gitmem/ is in .gitignore (prevents credential exposure)
+    try {
+        const projectRoot = process.cwd();
+        const gitignorePath = path.join(projectRoot, ".gitignore");
+        if (fs.existsSync(path.join(projectRoot, ".git"))) {
+            let gitignore = "";
+            if (fs.existsSync(gitignorePath)) {
+                gitignore = fs.readFileSync(gitignorePath, "utf-8");
+            }
+            if (!gitignore.split("\n").some(line => line.trim() === ".gitmem/" || line.trim() === ".gitmem")) {
+                const separator = gitignore.length > 0 && !gitignore.endsWith("\n") ? "\n" : "";
+                fs.appendFileSync(gitignorePath, `${separator}\n# GitMem local data (contains credentials)\n.gitmem/\n`);
+                console.log("  ✓ Added .gitmem/ to .gitignore");
             }
         }
-        // Show errors if any
-        if (totalErrors > 0) {
+    }
+    catch {
+        // Non-fatal — warn but don't block activation
+        console.log("  ⚠ Could not update .gitignore — manually add .gitmem/ to prevent credential exposure");
+    }
+    // Step 7: Migrate local data to Supabase (free → pro upgrade)
+    // Handles three scenarios:
+    //   A) Fresh upgrade: .json files exist → migrate and archive
+    //   B) Re-activation after failed migration: .pre-migration backups exist → reimport
+    //   C) Already migrated: neither exists → skip
+    if (supabaseUrl && supabaseKey && missingTables.length === 0) {
+        const hasLive = hasLocalData(gitmemDir);
+        const hasBackups = hasPreMigrationData(gitmemDir);
+        if (hasLive || hasBackups) {
+            if (hasLive) {
+                console.log("Migrating Local Data");
+                console.log("  Found existing local data from free tier...");
+            }
+            else {
+                console.log("Re-importing From Backups");
+                console.log("  Found .pre-migration backup files from a previous upgrade...");
+            }
             console.log("");
+            const migrationResult = hasLive
+                ? await migrateLocalToSupabase({
+                    supabaseUrl,
+                    supabaseKey,
+                    gitmemDir,
+                    onProgress: (msg) => console.log(msg),
+                })
+                : await reimportFromBackups({
+                    supabaseUrl,
+                    supabaseKey,
+                    gitmemDir,
+                    onProgress: (msg) => console.log(msg),
+                });
+            // Report results
+            const collections = Object.keys(migrationResult.migrated);
+            let totalMigrated = 0;
+            let totalSkipped = 0;
+            let totalErrors = 0;
             for (const col of collections) {
-                for (const err of migrationResult.errors[col] || []) {
-                    console.log(`  ⚠ ${col}: ${err}`);
+                const m = migrationResult.migrated[col];
+                const s = migrationResult.skipped[col];
+                const e = migrationResult.errors[col]?.length || 0;
+                totalMigrated += m;
+                totalSkipped += s;
+                totalErrors += e;
+                if (m > 0 || s > 0) {
+                    console.log(`  ${m > 0 ? "✓" : "⚠"} ${col}: ${m} migrated${s > 0 ? `, ${s} failed` : ""}`);
                 }
             }
-        }
-        if (totalMigrated > 0) {
-            console.log("");
-            console.log(`  ✓ Migrated ${totalMigrated} records to Supabase`);
-            // Archive local files so they aren't re-read
-            const archived = archiveLocalData(gitmemDir);
-            if (archived.length > 0) {
-                console.log(`  ✓ Local files archived (${archived.join(", ")}.json → .pre-migration)`);
+            // Show ALL errors
+            if (totalErrors > 0) {
+                console.log("");
+                for (const col of collections) {
+                    for (const err of migrationResult.errors[col] || []) {
+                        console.log(`  ⚠ ${col}: ${err}`);
+                    }
+                }
             }
+            if (totalMigrated > 0) {
+                console.log("");
+                console.log(`  ✓ Migrated ${totalMigrated} records to Supabase`);
+                if (hasLive) {
+                    // Archive local files so they aren't re-read
+                    const archived = archiveLocalData(gitmemDir);
+                    if (archived.length > 0) {
+                        console.log(`  ✓ Local files archived (${archived.join(", ")}.json → .pre-migration)`);
+                    }
+                }
+            }
+            else if (migrationResult.hasLocalData) {
+                console.log("  ⚠ Migration encountered errors. Local data preserved.");
+                console.log("    Check .gitmem/migration.log for details.");
+                console.log("    Fix issues and re-run: npx gitmem-mcp activate");
+            }
+            console.log("");
         }
-        else if (migrationResult.hasLocalData) {
-            console.log("  ⚠ Migration encountered errors. Local data preserved.");
-            console.log("    Re-run activate after resolving issues.");
-        }
-        console.log("");
     }
     // Summary
     console.log("");

package/dist/commands/migrate-local.d.ts

@@ -45,6 +45,24 @@ export declare function migrateLocalToSupabase(opts: {
     gitmemDir?: string;
     onProgress?: (msg: string) => void;
 }): Promise<MigrationResult>;
+/**
+ * Check if there are .pre-migration backup files that can be re-imported.
+ * This handles the case where a previous migration partially failed —
+ * the user runs `activate` again and we pick up from the backups.
+ */
+export declare function hasPreMigrationData(gitmemDir?: string): boolean;
+/**
+ * Re-import from .pre-migration backup files.
+ * Same as migrateLocalToSupabase but reads from *.json.pre-migration files.
+ * Idempotent: uses upsert (merge-duplicates) so re-running is safe.
+ */
+export declare function reimportFromBackups(opts: {
+    supabaseUrl: string;
+    supabaseKey: string;
+    tablePrefix?: string;
+    gitmemDir?: string;
+    onProgress?: (msg: string) => void;
+}): Promise<MigrationResult>;
 /**
  * Rename local collection files after successful migration
  * Adds .pre-migration suffix so data isn't lost but won't be re-read by free tier

package/dist/commands/migrate-local.js

@@ -27,6 +27,40 @@ const MIGRATABLE_COLLECTIONS = ["learnings", "sessions", "decisions", "scar_usag
 const STRIP_FIELDS = new Set(["is_starter"]);
 /** Fields that Supabase will reject if null (remove instead of sending null) */
 const NULLABLE_STRIP = new Set(["embedding"]);
+/**
+ * Known columns per table — only these fields are sent to Supabase.
+ * PostgREST rejects POSTs with unknown columns, so we must whitelist.
+ * This prevents accumulated local fields from different code versions
+ * from causing migration failures.
+ */
+const KNOWN_COLUMNS = {
+    learnings: new Set([
+        "id", "learning_type", "title", "description", "severity", "scar_type",
+        "counter_arguments", "problem_context", "solution_approach", "applies_when",
+        "keywords", "domain", "embedding", "project", "source_date",
+        "source_linear_issue", "persona_name", "why_this_matters",
+        "action_protocol", "self_check_criteria", "is_active",
+        "decay_multiplier", "repeat_mistake", "related_scar_id",
+        "repeat_mistake_details", "created_at", "updated_at",
+    ]),
+    sessions: new Set([
+        "id", "session_title", "session_date", "agent", "project",
+        "linear_issue", "recording_path", "transcript_path", "decisions",
+        "open_threads", "closing_reflection", "close_compliance",
+        "rapport_summary", "embedding", "created_at", "updated_at",
+    ]),
+    decisions: new Set([
+        "id", "decision_date", "title", "decision", "rationale",
+        "alternatives_considered", "personas_involved", "docs_affected",
+        "linear_issue", "session_id", "project", "embedding", "created_at",
+    ]),
+    scar_usage: new Set([
+        "id", "scar_id", "session_id", "agent", "issue_id",
+        "issue_identifier", "reference_type", "reference_context",
+        "surfaced_at", "acknowledged_at", "referenced",
+        "execution_successful", "variant_id", "created_at",
+    ]),
+};
 /**
  * Check if there is local data worth migrating
  */
@@ -62,23 +96,50 @@ function readLocalCollection(dir, collection) {
         return [];
     }
 }
+/**
+ * Columns that are TEXT in Supabase but may be stored as arrays locally.
+ * During migration, arrays are joined with newlines to fit the TEXT column.
+ * This prevents PostgREST 400 errors from type mismatches.
+ */
+const TEXT_COERCE_FIELDS = new Set([
+    "action_protocol",
+    "self_check_criteria",
+    "why_this_matters",
+]);
 /**
  * Clean a record for Supabase insertion:
  * - Strip local-only fields
  * - Remove null values for non-nullable columns
+ * - Only include fields that exist in the target table schema
+ * - Coerce arrays to strings for TEXT columns
  * - Ensure id exists
  */
-function cleanRecord(record) {
+function cleanRecord(record, collection) {
     if (!record.id)
         return null;
+    const knownCols = KNOWN_COLUMNS[collection];
     const cleaned = {};
+    const droppedFields = [];
     for (const [key, value] of Object.entries(record)) {
         if (STRIP_FIELDS.has(key))
             continue;
         if (NULLABLE_STRIP.has(key) && (value === null || value === undefined))
             continue;
+        // Only include fields that exist in the target table
+        if (knownCols && !knownCols.has(key)) {
+            droppedFields.push(key);
+            continue;
+        }
+        // Coerce arrays to strings for TEXT columns (schema says TEXT, code uses string[])
+        if (TEXT_COERCE_FIELDS.has(key) && Array.isArray(value)) {
+            cleaned[key] = value.join("\n");
+            continue;
+        }
         cleaned[key] = value;
     }
+    if (droppedFields.length > 0) {
+        console.error(`[migrate] Record ${String(record.id).substring(0, 8)}: dropped unknown fields: ${droppedFields.join(", ")}`);
+    }
     return cleaned;
 }
 /**
@@ -94,6 +155,14 @@ export async function migrateLocalToSupabase(opts) {
     const { supabaseUrl, supabaseKey, tablePrefix = "gitmem_", onProgress } = opts;
     const dir = opts.gitmemDir || getGitmemDir();
     const log = onProgress || ((msg) => console.log(msg));
+    // Migration log file — captures all details for debugging
+    const logLines = [];
+    const logFile = path.join(dir, "migration.log");
+    const mlog = (msg) => {
+        const ts = new Date().toISOString();
+        logLines.push(`[${ts}] ${msg}`);
+    };
+    mlog(`Migration started: ${supabaseUrl} (prefix: ${tablePrefix})`);
     const result = {
         migrated: {},
         skipped: {},
@@ -108,14 +177,21 @@ export async function migrateLocalToSupabase(opts) {
         result.migrated[collection] = 0;
         result.skipped[collection] = 0;
         result.errors[collection] = [];
-        if (records.length === 0)
+        if (records.length === 0) {
+            mlog(`${collection}: no local data`);
             continue;
+        }
         result.hasLocalData = true;
+        mlog(`${collection}: ${records.length} records to migrate → ${tableName}`);
         log(`  Migrating ${records.length} ${collection}...`);
         for (const record of records) {
-            const cleaned = cleanRecord(record);
+            const cleaned = cleanRecord(record, collection);
             if (!cleaned) {
                 result.skipped[collection]++;
+                const errMsg = `${String(record.id || "no-id").substring(0, 8)}: skipped (missing id)`;
+                result.errors[collection].push(errMsg);
+                mlog(`${collection} SKIP: ${errMsg}`);
+                result.total++;
                 continue;
             }
             try {
@@ -133,25 +209,89 @@ export async function migrateLocalToSupabase(opts) {
                 });
                 if (response.ok) {
                     result.migrated[collection]++;
+                    mlog(`${collection} OK: ${String(cleaned.id).substring(0, 8)} (${cleaned.title || ""})`);
                 }
                 else {
                     const text = await response.text();
-                    // Only log first 3 errors per collection to avoid spam
-                    if (result.errors[collection].length < 3) {
-                        result.errors[collection].push(`${String(cleaned.id).substring(0, 8)}: ${response.status} - ${text.substring(0, 100)}`);
-                    }
+                    const errMsg = `${String(cleaned.id).substring(0, 8)}: ${response.status} - ${text.substring(0, 200)}`;
+                    result.errors[collection].push(errMsg);
+                    mlog(`${collection} FAIL: ${errMsg}`);
+                    mlog(`${collection} FAIL payload: ${JSON.stringify(Object.keys(cleaned))}`);
                     result.skipped[collection]++;
                 }
             }
             catch (err) {
-                if (result.errors[collection].length < 3) {
-                    result.errors[collection].push(`${String(cleaned.id).substring(0, 8)}: ${err instanceof Error ? err.message : "Unknown error"}`);
-                }
+                const errMsg = `${String(cleaned.id).substring(0, 8)}: ${err instanceof Error ? err.message : "Unknown error"}`;
+                result.errors[collection].push(errMsg);
+                mlog(`${collection} ERROR: ${errMsg}`);
                 result.skipped[collection]++;
             }
             result.total++;
         }
     }
+    // Write migration log to disk for debugging
+    mlog(`Migration complete: migrated=${JSON.stringify(result.migrated)} skipped=${JSON.stringify(result.skipped)}`);
+    try {
+        fs.writeFileSync(logFile, logLines.join("\n") + "\n", "utf-8");
+        log(`  Migration log: ${logFile}`);
+    }
+    catch {
+        // Non-fatal — log file write failure shouldn't block migration
+    }
+    return result;
+}
+/**
+ * Check if there are .pre-migration backup files that can be re-imported.
+ * This handles the case where a previous migration partially failed —
+ * the user runs `activate` again and we pick up from the backups.
+ */
+export function hasPreMigrationData(gitmemDir) {
+    const dir = gitmemDir || getGitmemDir();
+    for (const collection of MIGRATABLE_COLLECTIONS) {
+        const backupPath = path.join(dir, `${collection}.json.pre-migration`);
+        if (fs.existsSync(backupPath)) {
+            try {
+                const data = JSON.parse(fs.readFileSync(backupPath, "utf-8"));
+                if (Array.isArray(data) && data.length > 0)
+                    return true;
+            }
+            catch {
+                // Corrupt backup
+            }
+        }
+    }
+    return false;
+}
+/**
+ * Re-import from .pre-migration backup files.
+ * Same as migrateLocalToSupabase but reads from *.json.pre-migration files.
+ * Idempotent: uses upsert (merge-duplicates) so re-running is safe.
+ */
+export async function reimportFromBackups(opts) {
+    const dir = opts.gitmemDir || getGitmemDir();
+    // Temporarily restore .pre-migration files as .json for the migration function
+    const restored = [];
+    for (const collection of MIGRATABLE_COLLECTIONS) {
+        const backupPath = path.join(dir, `${collection}.json.pre-migration`);
+        const livePath = path.join(dir, `${collection}.json`);
+        if (fs.existsSync(backupPath) && !fs.existsSync(livePath)) {
+            // Copy (not move) backup to live path for migration
+            fs.copyFileSync(backupPath, livePath);
+            restored.push(collection);
+        }
+    }
+    // Run migration
+    const result = await migrateLocalToSupabase(opts);
+    // Clean up: remove the restored copies (backups remain untouched)
+    for (const collection of restored) {
+        const livePath = path.join(dir, `${collection}.json`);
+        try {
+            fs.unlinkSync(livePath);
+        }
+        catch {
+            // Non-fatal
+        }
+    }
     return result;
 }
 /**

package/dist/services/embedding.d.ts

@@ -23,7 +23,13 @@ interface EmbeddingConfig {
     expectedDim: number;
 }
 /**
- * Detect the best available embedding provider from environment
+ * Detect the best available embedding provider from environment.
+ *
+ * Resolution chain for OpenRouter key:
+ *   1. process.env.OPENROUTER_API_KEY (env var)
+ *   2. .gitmem/config.json → openrouter_key (written by `activate`)
+ *
+ * This matches how supabase-client.ts resolves credentials via getProConfig().
  */
 export declare function detectProvider(): EmbeddingConfig;
 /**

package/dist/services/embedding.js

@@ -14,6 +14,7 @@
  * Records stored without embeddings can still be retrieved by ID/filters,
  * but won't appear in semantic search results.
  */
+import { getProConfig } from "./license.js";
 // Default embedding dimensions per provider
 const OPENAI_EMBEDDING_DIM = 1536;
 const OLLAMA_DEFAULT_DIM = 768;
@@ -38,10 +39,25 @@ function normalize(vec) {
     return vec.map((v) => v / magnitude);
 }
 /**
- * Detect the best available embedding provider from environment
+ * Detect the best available embedding provider from environment.
+ *
+ * Resolution chain for OpenRouter key:
+ *   1. process.env.OPENROUTER_API_KEY (env var)
+ *   2. .gitmem/config.json → openrouter_key (written by `activate`)
+ *
+ * This matches how supabase-client.ts resolves credentials via getProConfig().
  */
 export function detectProvider() {
     const forced = process.env.GITMEM_EMBEDDING_PROVIDER?.toLowerCase();
+    // Resolve OpenRouter key from env var OR config.json (same as supabase-client)
+    const resolveOpenRouterKey = () => {
+        if (process.env.OPENROUTER_API_KEY)
+            return process.env.OPENROUTER_API_KEY;
+        const config = getProConfig();
+        if (config.openrouterKey)
+            return config.openrouterKey;
+        return undefined;
+    };
     // Forced provider
     if (forced && forced !== "auto") {
         switch (forced) {
@@ -60,9 +76,9 @@ export function detectProvider() {
                 };
             }
             case "openrouter": {
-                const key = process.env.OPENROUTER_API_KEY;
+                const key = resolveOpenRouterKey();
                 if (!key) {
-                    console.warn("[embedding] GITMEM_EMBEDDING_PROVIDER=openrouter but OPENROUTER_API_KEY not set");
+                    console.warn("[embedding] GITMEM_EMBEDDING_PROVIDER=openrouter but no OpenRouter key found (env or config.json)");
                     return { provider: "none", apiUrl: "", apiKey: "", model: "", expectedDim: 0 };
                 }
                 return {
@@ -98,11 +114,13 @@ export function detectProvider() {
             expectedDim: OPENAI_EMBEDDING_DIM,
         };
     }
-    if (process.env.OPENROUTER_API_KEY) {
+    // Check env var AND config.json for OpenRouter key
+    const openrouterKey = resolveOpenRouterKey();
+    if (openrouterKey) {
         return {
             provider: "openrouter",
             apiUrl: OPENROUTER_API_URL,
-            apiKey: process.env.OPENROUTER_API_KEY,
+            apiKey: openrouterKey,
             model: OPENROUTER_EMBEDDING_MODEL,
             expectedDim: OPENAI_EMBEDDING_DIM,
         };

package/dist/services/transcript-chunker.js

@@ -7,6 +7,7 @@
  *
  */
 import { getTableName } from "./tier.js";
+import { getProConfig } from "./license.js";
 // OpenRouter API configuration (same as local-vector-search)
 const OPENROUTER_API_URL = "https://openrouter.ai/api/v1/embeddings";
 const EMBEDDING_MODEL = "openai/text-embedding-3-small";
@@ -32,9 +33,9 @@ function normalize(vec) {
  * Generate embedding using OpenRouter API
  */
 async function generateEmbedding(text) {
-    const apiKey = process.env.OPENROUTER_API_KEY;
+    const apiKey = process.env.OPENROUTER_API_KEY || getProConfig().openrouterKey;
     if (!apiKey) {
-        throw new Error("OPENROUTER_API_KEY not configured");
+        throw new Error("No OpenRouter key configured (set OPENROUTER_API_KEY env var or run: npx gitmem-mcp activate)");
     }
     const response = await fetch(OPENROUTER_API_URL, {
         method: "POST",

package/dist/services/variant-generation.js

@@ -17,6 +17,7 @@
  * Called fire-and-forget from create_learning — zero impact on UX latency.
  */
 import * as supabase from "./supabase-client.js";
+import { getProConfig } from "./license.js";
 // --- Pipeline versioning ---
 // Bump PIPELINE_VERSION when changing the prompt, model, or generation logic.
 // Format: "gen-{major}.{minor}" — major for prompt rewrites, minor for tweaks.
@@ -83,9 +84,9 @@ function buildUserPrompt(scar) {
  * Returns parsed output or null on failure.
  */
 async function generateWithLLM(scar) {
-    const apiKey = process.env.OPENROUTER_API_KEY;
+    const apiKey = process.env.OPENROUTER_API_KEY || getProConfig().openrouterKey;
     if (!apiKey) {
-        console.error("[variant-generation] No OPENROUTER_API_KEY — falling back to deterministic");
+        console.error("[variant-generation] No OpenRouter key (env or config.json) — falling back to deterministic");
         return null;
     }
     try {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gitmem-mcp",
-  "version": "1.6.0",
+  "version": "1.6.1",
   "mcpName": "io.github.gitmem-dev/gitmem",
   "description": "Persistent learning memory for AI coding agents. Memory that compounds.",
   "type": "module",