@nguyentamdat/mempalace 1.0.0

package/src/commands/mine.ts

@@ -0,0 +1,51 @@
+import { defineCommand } from "citty";
+import { resolvePalacePath } from "../cli";
+
+export default defineCommand({
+  meta: { description: "Mine files into the palace" },
+  args: {
+    dir: { type: "positional", description: "Directory to mine", required: true },
+    mode: {
+      type: "string",
+      description: "Ingest mode: 'projects' for code/docs, 'convos' for chat exports",
+      default: "projects",
+    },
+    wing: { type: "string", description: "Wing name (default: directory name)" },
+    agent: { type: "string", description: "Your name — recorded on every drawer", default: "mempalace" },
+    limit: { type: "string", description: "Max files to process (0 = all)", default: "0" },
+    "dry-run": { type: "boolean", description: "Show what would be filed without filing", default: false },
+    extract: {
+      type: "string",
+      description: "Extraction strategy for convos: 'exchange' or 'general'",
+      default: "exchange",
+    },
+  },
+  async run({ args }) {
+    const palacePath = resolvePalacePath(args.palace as string | undefined);
+    const limit = parseInt(args.limit, 10);
+    const dryRun = args["dry-run"];
+
+    if (args.mode === "convos") {
+      const { mineConvos } = await import("../convo-miner");
+      await mineConvos({
+        convoDir: args.dir,
+        palacePath,
+        wing: args.wing,
+        agent: args.agent,
+        limit,
+        dryRun,
+        extractMode: args.extract as "exchange" | "general",
+      });
+    } else {
+      const { mine } = await import("../miner");
+      await mine({
+        projectDir: args.dir,
+        palacePath,
+        wingOverride: args.wing,
+        agent: args.agent,
+        limit,
+        dryRun,
+      });
+    }
+  },
+});

package/src/commands/search.ts

@@ -0,0 +1,23 @@
+import { defineCommand } from "citty";
+import { resolvePalacePath } from "../cli";
+
+export default defineCommand({
+  meta: { description: "Find anything, exact words" },
+  args: {
+    query: { type: "positional", description: "What to search for", required: true },
+    wing: { type: "string", description: "Limit to one project" },
+    room: { type: "string", description: "Limit to one room" },
+    results: { type: "string", description: "Number of results", default: "5" },
+  },
+  async run({ args }) {
+    const { search } = await import("../searcher");
+    const palacePath = resolvePalacePath(args.palace as string | undefined);
+    await search({
+      query: args.query,
+      palacePath,
+      wing: args.wing,
+      room: args.room,
+      nResults: parseInt(args.results, 10),
+    });
+  },
+});

package/src/commands/split.ts

@@ -0,0 +1,20 @@
+import { defineCommand } from "citty";
+
+export default defineCommand({
+  meta: { description: "Split concatenated transcript mega-files into per-session files" },
+  args: {
+    dir: { type: "positional", description: "Directory containing transcript files", required: true },
+    "output-dir": { type: "string", description: "Write split files here (default: same as source)" },
+    "dry-run": { type: "boolean", description: "Show what would be split without writing", default: false },
+    "min-sessions": { type: "string", description: "Only split files with at least N sessions", default: "2" },
+  },
+  async run({ args }) {
+    const { splitMegaFiles } = await import("../split-mega-files");
+    splitMegaFiles({
+      dir: args.dir,
+      outputDir: args["output-dir"],
+      dryRun: args["dry-run"],
+      minSessions: parseInt(args["min-sessions"], 10),
+    });
+  },
+});

package/src/commands/status.ts

@@ -0,0 +1,12 @@
+import { defineCommand } from "citty";
+import { resolvePalacePath } from "../cli";
+
+export default defineCommand({
+  meta: { description: "Show what's been filed" },
+  args: {},
+  async run({ args }) {
+    const { status } = await import("../miner");
+    const palacePath = resolvePalacePath(args.palace as string | undefined);
+    await status(palacePath);
+  },
+});

package/src/commands/wake-up.ts

@@ -0,0 +1,20 @@
+import { defineCommand } from "citty";
+import { resolvePalacePath } from "../cli";
+
+export default defineCommand({
+  meta: { description: "Show L0 + L1 wake-up context (~600-900 tokens)" },
+  args: {
+    wing: { type: "string", description: "Wake-up for a specific project/wing" },
+  },
+  async run({ args }) {
+    const { MemoryStack } = await import("../layers");
+    const palacePath = resolvePalacePath(args.palace as string | undefined);
+    const stack = new MemoryStack(palacePath);
+
+    const text = await stack.wakeUp(args.wing);
+    const tokens = Math.floor(text.length / 4);
+    console.log(`Wake-up text (~${tokens} tokens):`);
+    console.log("=".repeat(50));
+    console.log(text);
+  },
+});

package/src/config.ts

@@ -0,0 +1,111 @@
+/**
+ * MemPalace configuration system.
+ *
+ * Priority: env vars > config file (~/.mempalace/config.json) > defaults
+ */
+
+import { existsSync, readFileSync, writeFileSync, mkdirSync } from "fs";
+import { join } from "path";
+import { homedir } from "os";
+
+export const DEFAULT_PALACE_PATH = join(homedir(), ".mempalace", "palace");
+export const DEFAULT_COLLECTION_NAME = "mempalace_drawers";
+
+export const DEFAULT_TOPIC_WINGS = [
+  "emotions",
+  "consciousness",
+  "memory",
+  "technical",
+  "identity",
+  "family",
+  "creative",
+];
+
+export const DEFAULT_HALL_KEYWORDS: Record<string, string[]> = {
+  emotions: [
+    "scared", "afraid", "worried", "happy", "sad",
+    "love", "hate", "feel", "cry", "tears",
+  ],
+  consciousness: [
+    "consciousness", "conscious", "aware", "real",
+    "genuine", "soul", "exist", "alive",
+  ],
+  memory: ["memory", "remember", "forget", "recall", "archive", "palace", "store"],
+  technical: [
+    "code", "python", "script", "bug", "error",
+    "function", "api", "database", "server",
+  ],
+  identity: ["identity", "name", "who am i", "persona", "self"],
+  family: ["family", "kids", "children", "daughter", "son", "parent", "mother", "father"],
+  creative: ["game", "gameplay", "player", "app", "design", "art", "music", "story"],
+};
+
+function readJsonFile(path: string): Record<string, unknown> | null {
+  try {
+    if (existsSync(path)) {
+      return JSON.parse(readFileSync(path, "utf-8"));
+    }
+  } catch {
+    // Ignore parse/read errors
+  }
+  return null;
+}
+
+export class MempalaceConfig {
+  private configDir: string;
+  private configFile: string;
+  private peopleMapFile: string;
+  private fileConfig: Record<string, unknown>;
+
+  constructor(configDir?: string) {
+    this.configDir = configDir ?? join(homedir(), ".mempalace");
+    this.configFile = join(this.configDir, "config.json");
+    this.peopleMapFile = join(this.configDir, "people_map.json");
+    this.fileConfig = readJsonFile(this.configFile) ?? {};
+  }
+
+  get palacePath(): string {
+    const envVal =
+      process.env.MEMPALACE_PALACE_PATH ?? process.env.MEMPAL_PALACE_PATH;
+    if (envVal) return envVal;
+    return (this.fileConfig.palace_path as string) ?? DEFAULT_PALACE_PATH;
+  }
+
+  get collectionName(): string {
+    return (this.fileConfig.collection_name as string) ?? DEFAULT_COLLECTION_NAME;
+  }
+
+  get peopleMap(): Record<string, string> {
+    const fromFile = readJsonFile(this.peopleMapFile);
+    if (fromFile) return fromFile as Record<string, string>;
+    return (this.fileConfig.people_map as Record<string, string>) ?? {};
+  }
+
+  get topicWings(): string[] {
+    return (this.fileConfig.topic_wings as string[]) ?? DEFAULT_TOPIC_WINGS;
+  }
+
+  get hallKeywords(): Record<string, string[]> {
+    return (this.fileConfig.hall_keywords as Record<string, string[]>) ?? DEFAULT_HALL_KEYWORDS;
+  }
+
+  init(): string {
+    mkdirSync(this.configDir, { recursive: true });
+    if (!existsSync(this.configFile)) {
+      const defaultConfig = {
+        palace_path: DEFAULT_PALACE_PATH,
+        collection_name: DEFAULT_COLLECTION_NAME,
+        topic_wings: DEFAULT_TOPIC_WINGS,
+        hall_keywords: DEFAULT_HALL_KEYWORDS,
+      };
+      writeFileSync(this.configFile, JSON.stringify(defaultConfig, null, 2));
+    }
+    return this.configFile;
+  }
+
+  savePeopleMap(peopleMap: Record<string, string>): string {
+    mkdirSync(this.configDir, { recursive: true });
+    writeFileSync(this.peopleMapFile, JSON.stringify(peopleMap, null, 2));
+    return this.peopleMapFile;
+  }
+}

package/src/convo-miner.ts

@@ -0,0 +1,373 @@
+import { createHash } from "node:crypto";
+import { mkdirSync, readdirSync, statSync } from "node:fs";
+import { basename, extname, resolve } from "node:path";
+import { ChromaClient } from "chromadb";
+
+import { extractMemories } from "./general-extractor";
+import { normalize } from "./normalize";
+
+export const CONVO_EXTENSIONS = new Set([".txt", ".md", ".json", ".jsonl"]);
+
+export const SKIP_DIRS = new Set([
+  ".git",
+  "node_modules",
+  "__pycache__",
+  ".venv",
+  "venv",
+  "env",
+  "dist",
+  "build",
+  ".next",
+  ".mempalace",
+]);
+
+export const MIN_CHUNK_SIZE = 30;
+
+export const TOPIC_KEYWORDS: Record<string, string[]> = {
+  technical: ["code", "python", "function", "bug", "error", "api", "database", "server", "deploy", "git", "test", "debug", "refactor"],
+  architecture: ["architecture", "design", "pattern", "structure", "schema", "interface", "module", "component", "service", "layer"],
+  planning: ["plan", "roadmap", "milestone", "deadline", "priority", "sprint", "backlog", "scope", "requirement", "spec"],
+  decisions: ["decided", "chose", "picked", "switched", "migrated", "replaced", "trade-off", "alternative", "option", "approach"],
+  problems: ["problem", "issue", "broken", "failed", "crash", "stuck", "workaround", "fix", "solved", "resolved"],
+};
+
+const COLLECTION_NAME = "mempalace_drawers";
+
+type ExtractMode = "exchange" | "general";
+
+type Chunk = {
+  content: string;
+  chunk_index: number;
+  memory_type?: string;
+};
+
+type MineConvosOptions = {
+  convoDir: string;
+  palacePath: string;
+  wing?: string;
+  agent?: string;
+  limit?: number;
+  dryRun?: boolean;
+  extractMode?: ExtractMode;
+};
+
+type DrawerCollection = Awaited<ReturnType<ChromaClient["getCollection"]>>;
+
+function formatRule(char: string, width = 55): string {
+  return char.repeat(width);
+}
+
+export function chunkExchanges(content: string): Chunk[] {
+  const lines = content.split("\n");
+  const quoteLines = lines.filter((line) => line.trim().startsWith(">")).length;
+
+  if (quoteLines >= 3) {
+    return chunkByExchange(lines);
+  }
+
+  return chunkByParagraph(content);
+}
+
+function chunkByExchange(lines: string[]): Chunk[] {
+  const chunks: Chunk[] = [];
+  let index = 0;
+
+  while (index < lines.length) {
+    const line = lines[index];
+    if (line.trim().startsWith(">")) {
+      const userTurn = line.trim();
+      index += 1;
+
+      const aiLines: string[] = [];
+      while (index < lines.length) {
+        const nextLine = lines[index];
+        const stripped = nextLine.trim();
+
+        if (stripped.startsWith(">") || stripped.startsWith("---")) {
+          break;
+        }
+
+        if (stripped) {
+          aiLines.push(stripped);
+        }
+
+        index += 1;
+      }
+
+      const aiResponse = aiLines.slice(0, 8).join(" ");
+      const chunkContent = aiResponse ? `${userTurn}\n${aiResponse}` : userTurn;
+
+      if (chunkContent.trim().length > MIN_CHUNK_SIZE) {
+        chunks.push({
+          content: chunkContent,
+          chunk_index: chunks.length,
+        });
+      }
+    } else {
+      index += 1;
+    }
+  }
+
+  return chunks;
+}
+
+function chunkByParagraph(content: string): Chunk[] {
+  const chunks: Chunk[] = [];
+  const paragraphs = content
+    .split("\n\n")
+    .map((paragraph) => paragraph.trim())
+    .filter((paragraph) => paragraph);
+
+  if (paragraphs.length <= 1 && content.split("\n").length - 1 > 20) {
+    const lines = content.split("\n");
+    for (let index = 0; index < lines.length; index += 25) {
+      const group = lines.slice(index, index + 25).join("\n").trim();
+      if (group.length > MIN_CHUNK_SIZE) {
+        chunks.push({ content: group, chunk_index: chunks.length });
+      }
+    }
+    return chunks;
+  }
+
+  for (const paragraph of paragraphs) {
+    if (paragraph.length > MIN_CHUNK_SIZE) {
+      chunks.push({ content: paragraph, chunk_index: chunks.length });
+    }
+  }
+
+  return chunks;
+}
+
+export function detectConvoRoom(content: string): string {
+  const contentLower = content.slice(0, 3000).toLowerCase();
+  const scores = new Map<string, number>();
+
+  for (const [room, keywords] of Object.entries(TOPIC_KEYWORDS)) {
+    const score = keywords.reduce((total, keyword) => total + (contentLower.includes(keyword) ? 1 : 0), 0);
+    if (score > 0) {
+      scores.set(room, score);
+    }
+  }
+
+  let bestRoom = "general";
+  let bestScore = 0;
+  for (const [room, score] of scores.entries()) {
+    if (score > bestScore) {
+      bestRoom = room;
+      bestScore = score;
+    }
+  }
+
+  return bestScore > 0 ? bestRoom : "general";
+}
+
+export async function getCollection(palacePath: string): Promise<DrawerCollection> {
+  mkdirSync(palacePath, { recursive: true });
+  const client = new ChromaClient();
+  return client.getOrCreateCollection({ name: COLLECTION_NAME });
+}
+
+export async function fileAlreadyMined(collection: DrawerCollection, sourceFile: string): Promise<boolean> {
+  try {
+    const results = await collection.get({ where: { source_file: sourceFile }, limit: 1 });
+    return results.ids.length > 0;
+  } catch {
+    return false;
+  }
+}
+
+export function scanConvos(convoDir: string): string[] {
+  const convoPath = resolve(convoDir.replace(/^~(?=$|\/)/, process.env.HOME ?? "~"));
+  const files: string[] = [];
+
+  const walk = (currentDir: string) => {
+    for (const entry of readdirSync(currentDir, { withFileTypes: true })) {
+      if (entry.isDirectory()) {
+        if (!SKIP_DIRS.has(entry.name)) {
+          walk(`${currentDir}/${entry.name}`);
+        }
+        continue;
+      }
+
+      if (!entry.isFile()) {
+        continue;
+      }
+
+      const filepath = `${currentDir}/${entry.name}`;
+      if (!CONVO_EXTENSIONS.has(extname(filepath).toLowerCase())) {
+        continue;
+      }
+
+      try {
+        statSync(filepath);
+        files.push(filepath);
+      } catch {
+      }
+    }
+  };
+
+  walk(convoPath);
+  return files;
+}
+
+export async function mineConvos({
+  convoDir,
+  palacePath,
+  wing,
+  agent = "mempalace",
+  limit = 0,
+  dryRun = false,
+  extractMode = "exchange",
+}: MineConvosOptions): Promise<void> {
+  const convoPath = resolve(convoDir.replace(/^~(?=$|\/)/, process.env.HOME ?? "~"));
+  const resolvedWing = wing ?? basename(convoPath).toLowerCase().replace(/ /g, "_").replace(/-/g, "_");
+
+  let files = scanConvos(convoDir);
+  if (limit > 0) {
+    files = files.slice(0, limit);
+  }
+
+  console.log(`\n${formatRule("=")}`);
+  console.log("  MemPalace Mine — Conversations");
+  console.log(formatRule("="));
+  console.log(`  Wing:    ${resolvedWing}`);
+  console.log(`  Source:  ${convoPath}`);
+  console.log(`  Files:   ${files.length}`);
+  console.log(`  Palace:  ${palacePath}`);
+  if (dryRun) {
+    console.log("  DRY RUN — nothing will be filed");
+  }
+  console.log(`${formatRule("─")}\n`);
+
+  const collection = dryRun ? null : await getCollection(palacePath);
+
+  let totalDrawers = 0;
+  let filesSkipped = 0;
+  const roomCounts = new Map<string, number>();
+
+  for (const [index, filepath] of files.entries()) {
+    const sourceFile = filepath;
+
+    if (!dryRun) {
+      if (collection === null) {
+        throw new Error("Collection is required when dryRun is false");
+      }
+      if (await fileAlreadyMined(collection, sourceFile)) {
+        filesSkipped += 1;
+        continue;
+      }
+    }
+
+    let content: string;
+    try {
+      content = await normalize(sourceFile);
+    } catch {
+      continue;
+    }
+
+    if (!content.trim() || content.trim().length < MIN_CHUNK_SIZE) {
+      continue;
+    }
+
+    const chunks = extractMode === "general" ? extractMemories(content) : chunkExchanges(content);
+    if (chunks.length === 0) {
+      continue;
+    }
+
+    const room = extractMode !== "general" ? detectConvoRoom(content) : null;
+
+    if (dryRun) {
+      if (extractMode === "general") {
+        const typeCounts = new Map<string, number>();
+        for (const chunk of chunks) {
+          const memoryType = chunk.memory_type ?? "general";
+          typeCounts.set(memoryType, (typeCounts.get(memoryType) ?? 0) + 1);
+        }
+        const typesStr = [...typeCounts.entries()]
+          .sort((left, right) => right[1] - left[1])
+          .map(([type, count]) => `${type}:${count}`)
+          .join(", ");
+        console.log(`    [DRY RUN] ${basename(filepath)} → ${chunks.length} memories (${typesStr})`);
+      } else {
+        console.log(`    [DRY RUN] ${basename(filepath)} → room:${room} (${chunks.length} drawers)`);
+      }
+
+      totalDrawers += chunks.length;
+      if (extractMode === "general") {
+        for (const chunk of chunks) {
+          const memoryType = chunk.memory_type ?? "general";
+          roomCounts.set(memoryType, (roomCounts.get(memoryType) ?? 0) + 1);
+        }
+      } else if (room !== null) {
+        roomCounts.set(room, (roomCounts.get(room) ?? 0) + 1);
+      }
+      continue;
+    }
+
+    if (extractMode !== "general" && room !== null) {
+      roomCounts.set(room, (roomCounts.get(room) ?? 0) + 1);
+    }
+
+    let drawersAdded = 0;
+    for (const chunk of chunks) {
+      const chunkRoom = extractMode === "general" ? (chunk.memory_type ?? "general") : (room ?? "general");
+      if (extractMode === "general") {
+        roomCounts.set(chunkRoom, (roomCounts.get(chunkRoom) ?? 0) + 1);
+      }
+
+      const drawerId = `drawer_${resolvedWing}_${chunkRoom}_${createHash("md5")
+        .update(sourceFile + String(chunk.chunk_index))
+        .digest("hex")
+        .slice(0, 16)}`;
+
+      if (collection === null) {
+        throw new Error("Collection is required when dryRun is false");
+      }
+
+      try {
+        await collection.add({
+          documents: [chunk.content],
+          ids: [drawerId],
+          metadatas: [
+            {
+              wing: resolvedWing,
+              room: chunkRoom,
+              source_file: sourceFile,
+              chunk_index: chunk.chunk_index,
+              added_by: agent,
+              filed_at: new Date().toISOString(),
+              ingest_mode: "convos",
+              extract_mode: extractMode,
+            },
+          ],
+        });
+        drawersAdded += 1;
+      } catch (error) {
+        if (!(error instanceof Error) || !error.message.toLowerCase().includes("already exists")) {
+          throw error;
+        }
+      }
+    }
+
+    totalDrawers += drawersAdded;
+    console.log(
+      `  ✓ [${String(index + 1).padStart(4)}/${files.length}] ${basename(filepath)
+        .slice(0, 50)
+        .padEnd(50)} +${drawersAdded}`,
+    );
+  }
+
+  console.log(`\n${formatRule("=")}`);
+  console.log("  Done.");
+  console.log(`  Files processed: ${files.length - filesSkipped}`);
+  console.log(`  Files skipped (already filed): ${filesSkipped}`);
+  console.log(`  Drawers filed: ${totalDrawers}`);
+  if (roomCounts.size > 0) {
+    console.log("\n  By room:");
+    for (const [roomName, count] of [...roomCounts.entries()].sort((left, right) => right[1] - left[1])) {
+      console.log(`    ${roomName.padEnd(20)} ${count} files`);
+    }
+  }
+  console.log('\n  Next: mempalace search "what you\'re looking for"');
+  console.log(`${formatRule("=")}\n`);
+}