@nguyentamdat/mempalace 1.0.0

package/src/room-detector.ts

@@ -0,0 +1,6 @@
+export {
+	detectRoomsFromFiles,
+	detectRoomsFromFolders,
+	detectRoomsLocal,
+	saveConfig,
+} from "./room-detector-local";

package/src/searcher.ts

@@ -0,0 +1,181 @@
+import { basename } from "node:path";
+import { ChromaClient, DefaultEmbeddingFunction } from "chromadb";
+import { MempalaceConfig } from "./config";
+
+type SearchOptions = {
+  query: string;
+  palacePath: string;
+  wing?: string;
+  room?: string;
+  nResults?: number;
+};
+
+type SearchHit = {
+  text: string;
+  wing: string;
+  room: string;
+  source_file: string;
+  similarity: number;
+};
+
+type DrawerCollection = Awaited<ReturnType<ChromaClient["getCollection"]>>;
+type DrawerQueryParams = Parameters<DrawerCollection["query"]>[0];
+
+function buildWhereFilter(wing?: string, room?: string) {
+  let where: Record<string, unknown> = {};
+
+  if (wing && room) {
+    where = { $and: [{ wing }, { room }] };
+  } else if (wing) {
+    where = { wing };
+  } else if (room) {
+    where = { room };
+  }
+
+  return where;
+}
+
+async function getDrawerCollection(palacePath: string) {
+  const config = new MempalaceConfig();
+  const collectionName = config.collectionName;
+  const client = new ChromaClient({ path: palacePath || config.palacePath });
+  const embeddingFunction = new DefaultEmbeddingFunction();
+
+  try {
+    return await client.getCollection({ name: collectionName, embeddingFunction });
+  } catch {
+    throw new Error(`No palace found at ${palacePath || config.palacePath}`);
+  }
+}
+
+export async function search({
+  query,
+  palacePath,
+  wing,
+  room,
+  nResults = 5,
+}: SearchOptions): Promise<void> {
+  let col: DrawerCollection;
+
+  try {
+    col = await getDrawerCollection(palacePath);
+  } catch {
+    console.log(`\n  No palace found at ${palacePath}`);
+    console.log("  Run: mempalace init <dir> then mempalace mine <dir>");
+    process.exit(1);
+  }
+
+  const where = buildWhereFilter(wing, room);
+
+  try {
+    const kwargs = {
+      query_texts: [query],
+      n_results: nResults,
+      include: ["documents", "metadatas", "distances"],
+    } as unknown as DrawerQueryParams;
+
+    if (Object.keys(where).length > 0) {
+      kwargs.where = where;
+    }
+
+    const results = await col.query(kwargs);
+    const docs = results.documents?.[0] ?? [];
+    const metas = results.metadatas?.[0] ?? [];
+    const dists = results.distances?.[0] ?? [];
+
+    if (docs.length === 0) {
+      console.log(`\n  No results found for: "${query}"`);
+      return;
+    }
+
+    console.log(`\n${"=".repeat(60)}`);
+    console.log(`  Results for: "${query}"`);
+    if (wing) console.log(`  Wing: ${wing}`);
+    if (room) console.log(`  Room: ${room}`);
+    console.log(`${"=".repeat(60)}\n`);
+
+    for (let i = 0; i < docs.length; i++) {
+      const doc = String(docs[i] ?? "");
+      const meta = (metas[i] ?? {}) as Record<string, unknown>;
+      const dist = Number(dists[i] ?? 0);
+      const similarity = Math.round((1 - dist) * 1000) / 1000;
+      const source = basename(String(meta.source_file ?? "?"));
+      const wingName = String(meta.wing ?? "?");
+      const roomName = String(meta.room ?? "?");
+
+      console.log(`  [${i + 1}] ${wingName} / ${roomName}`);
+      console.log(`      Source: ${source}`);
+      console.log(`      Match:  ${similarity}`);
+      console.log();
+
+      for (const line of doc.trim().split("\n")) {
+        console.log(`      ${line}`);
+      }
+      console.log();
+      console.log(`  ${"─".repeat(56)}`);
+    }
+
+    console.log();
+  } catch (error) {
+    console.log(`\n  Search error: ${error}`);
+    process.exit(1);
+  }
+}
+
+export async function searchMemories({
+  query,
+  palacePath,
+  wing,
+  room,
+  nResults = 5,
+}: SearchOptions): Promise<{ query?: string; filters?: { wing?: string; room?: string }; results?: SearchHit[]; error?: string }> {
+  let col: DrawerCollection;
+
+  try {
+    col = await getDrawerCollection(palacePath);
+  } catch (error) {
+    return { error: `No palace found at ${palacePath}: ${error}` };
+  }
+
+  const where = buildWhereFilter(wing, room);
+
+  try {
+    const kwargs = {
+      query_texts: [query],
+      n_results: nResults,
+      include: ["documents", "metadatas", "distances"],
+    } as unknown as DrawerQueryParams;
+
+    if (Object.keys(where).length > 0) {
+      kwargs.where = where;
+    }
+
+    const results = await col.query(kwargs);
+    const docs = results.documents?.[0] ?? [];
+    const metas = results.metadatas?.[0] ?? [];
+    const dists = results.distances?.[0] ?? [];
+
+    const hits: SearchHit[] = [];
+    for (let i = 0; i < docs.length; i++) {
+      const doc = String(docs[i] ?? "");
+      const meta = (metas[i] ?? {}) as Record<string, unknown>;
+      const dist = Number(dists[i] ?? 0);
+
+      hits.push({
+        text: doc,
+        wing: String(meta.wing ?? "unknown"),
+        room: String(meta.room ?? "unknown"),
+        source_file: basename(String(meta.source_file ?? "?")),
+        similarity: Math.round((1 - dist) * 1000) / 1000,
+      });
+    }
+
+    return {
+      query,
+      filters: { wing, room },
+      results: hits,
+    };
+  } catch (error) {
+    return { error: `Search error: ${error}` };
+  }
+}

package/src/spellcheck.ts

@@ -0,0 +1,200 @@
+import { existsSync, readFileSync } from "node:fs";
+import { EntityRegistry } from "./entity-registry";
+
+const SYSTEM_DICT_PATH = "/usr/share/dict/words";
+const HAS_DIGIT = /\d/u;
+const IS_CAMEL = /[A-Z][a-z]+[A-Z]/u;
+const IS_ALLCAPS = /^(?:[A-Z_@#$%^&*()+={}|<>?.:/\\]|\[|\])+$/u;
+const IS_TECHNICAL = /[-_]/u;
+const IS_URL = /https?:\/\/|www\.|\/Users\/|~\/|\.[a-z]{2,4}$/iu;
+const IS_CODE_OR_EMOJI = /[`*_#{}[\]\\]/u;
+const TOKEN_RE = /(\S+)/gu;
+const TRAILING_PUNCTUATION_RE = /[.,!?;:'")]+$/u;
+const MIN_LENGTH = 4;
+
+type Speller = (token: string) => string;
+
+let systemWordsCache: ReadonlySet<string> | null = null;
+
+function defaultSpeller(token: string): string {
+  // TODO: Plug in a Bun-compatible spellchecker when we choose one.
+  return token;
+}
+
+function shouldSkip(token: string, knownNames: ReadonlySet<string>): boolean {
+  if (token.length < MIN_LENGTH) {
+    return true;
+  }
+  if (HAS_DIGIT.test(token)) {
+    return true;
+  }
+  if (IS_CAMEL.test(token)) {
+    return true;
+  }
+  if (IS_ALLCAPS.test(token)) {
+    return true;
+  }
+  if (IS_TECHNICAL.test(token)) {
+    return true;
+  }
+  if (IS_URL.test(token)) {
+    return true;
+  }
+  if (IS_CODE_OR_EMOJI.test(token)) {
+    return true;
+  }
+  if (knownNames.has(token.toLowerCase())) {
+    return true;
+  }
+  return false;
+}
+
+function loadKnownNames(): Set<string> {
+  try {
+    const registry = EntityRegistry.load();
+    const names = new Set<string>();
+
+    for (const [canonical, info] of Object.entries(registry.people)) {
+      if (canonical) {
+        names.add(canonical.toLowerCase());
+      }
+
+      for (const alias of info.aliases) {
+        names.add(alias.toLowerCase());
+      }
+
+      if (typeof info.canonical === "string" && info.canonical) {
+        names.add(info.canonical.toLowerCase());
+      }
+    }
+
+    for (const project of registry.projects) {
+      names.add(project.toLowerCase());
+    }
+
+    return names;
+  } catch {
+    return new Set<string>();
+  }
+}
+
+function editDistance(a: string, b: string): number {
+  if (a === b) {
+    return 0;
+  }
+  if (a.length === 0) {
+    return b.length;
+  }
+  if (b.length === 0) {
+    return a.length;
+  }
+
+  let previousRow = Array.from({ length: b.length + 1 }, (_, index) => index);
+
+  for (let i = 0; i < a.length; i += 1) {
+    const currentRow = [i + 1];
+
+    for (let j = 0; j < b.length; j += 1) {
+      const substitutionCost = a[i] === b[j] ? 0 : 1;
+      currentRow.push(
+        Math.min(
+          previousRow[j + 1] + 1,
+          currentRow[j] + 1,
+          previousRow[j] + substitutionCost,
+        ),
+      );
+    }
+
+    previousRow = currentRow;
+  }
+
+  return previousRow[previousRow.length - 1];
+}
+
+function getSystemWords(): ReadonlySet<string> {
+  if (systemWordsCache !== null) {
+    return systemWordsCache;
+  }
+
+  if (!existsSync(SYSTEM_DICT_PATH)) {
+    systemWordsCache = new Set<string>();
+    return systemWordsCache;
+  }
+
+  const words = readFileSync(SYSTEM_DICT_PATH, "utf-8")
+    .split(/\r?\n/u)
+    .map((word) => word.trim().toLowerCase())
+    .filter((word) => word.length > 0);
+
+  systemWordsCache = new Set(words);
+  return systemWordsCache;
+}
+
+function splitTrailingPunctuation(token: string): { stripped: string; punctuation: string } {
+  const punctuation = token.match(TRAILING_PUNCTUATION_RE)?.[0] ?? "";
+  if (punctuation.length === 0) {
+    return { stripped: token, punctuation: "" };
+  }
+  return {
+    stripped: token.slice(0, token.length - punctuation.length),
+    punctuation,
+  };
+}
+
+export function spellcheckUserText(
+  text: string,
+  knownNames?: ReadonlySet<string>,
+  speller: Speller = defaultSpeller,
+): string {
+  const resolvedKnownNames = knownNames ?? loadKnownNames();
+  const systemWords = getSystemWords();
+
+  return text.replace(TOKEN_RE, (token) => {
+    const { stripped, punctuation } = splitTrailingPunctuation(token);
+    if (!stripped || shouldSkip(stripped, resolvedKnownNames)) {
+      return token;
+    }
+
+    if (stripped[0] !== stripped[0].toLowerCase()) {
+      return token;
+    }
+
+    if (systemWords.has(stripped.toLowerCase())) {
+      return token;
+    }
+
+    const corrected = speller(stripped);
+    if (corrected !== stripped) {
+      const distance = editDistance(stripped, corrected);
+      const maxEdits = stripped.length <= 7 ? 2 : 3;
+      if (distance > maxEdits) {
+        return token;
+      }
+    }
+
+    return `${corrected}${punctuation}`;
+  });
+}
+
+export function spellcheckTranscriptLine(line: string): string {
+  const stripped = line.trimStart();
+  if (!stripped.startsWith(">")) {
+    return line;
+  }
+
+  const leadingWhitespaceLength = line.length - stripped.length;
+  const prefixLength = leadingWhitespaceLength + (stripped.startsWith("> ") ? 2 : 1);
+  const message = line.slice(prefixLength);
+  if (message.trim().length === 0) {
+    return line;
+  }
+
+  return `${line.slice(0, prefixLength)}${spellcheckUserText(message)}`;
+}
+
+export function spellcheckTranscript(content: string): string {
+  return content
+    .split("\n")
+    .map((line) => spellcheckTranscriptLine(line))
+    .join("\n");
+}

package/src/split-mega-files.d.ts

@@ -0,0 +1,8 @@
+export type SplitMegaFilesOptions = {
+  dir: string;
+  outputDir?: string;
+  dryRun?: boolean;
+  minSessions: number;
+};
+
+export declare function splitMegaFiles(options: SplitMegaFilesOptions): void;

package/src/split-mega-files.ts

@@ -0,0 +1,297 @@
+import {
+  existsSync,
+  mkdirSync,
+  readdirSync,
+  readFileSync,
+  renameSync,
+  statSync,
+  writeFileSync,
+} from "node:fs";
+import { homedir } from "node:os";
+import { basename, join, parse } from "node:path";
+
+const HOME = homedir();
+const LUMI_DIR = process.env.MEMPALACE_SOURCE_DIR ?? join(HOME, "Desktop/transcripts");
+const KNOWN_NAMES_PATH = join(HOME, ".mempalace", "known_names.json");
+const FALLBACK_PEOPLE = ["Alice", "Ben", "Riley", "Max", "Sam", "Devon", "Jordan"];
+const TIMESTAMP_PATTERN = /⏺\s+(\d{1,2}:\d{2}\s+[AP]M)\s+\w+,\s+(\w+)\s+(\d{1,2}),\s+(\d{4})/;
+const MONTHS: Record<string, string> = {
+  January: "01",
+  February: "02",
+  March: "03",
+  April: "04",
+  May: "05",
+  June: "06",
+  July: "07",
+  August: "08",
+  September: "09",
+  October: "10",
+  November: "11",
+  December: "12",
+};
+const SUBJECT_SKIP_PATTERN = /^(\.\/|cd |ls |python|bash|git |cat |source |export |claude|\.\/activate)/;
+
+type KnownNamesConfig = {
+  names?: unknown;
+  username_map?: unknown;
+};
+
+type SplitMegaFilesOptions = {
+  dir?: string;
+  outputDir?: string;
+  dryRun?: boolean;
+  minSessions?: number;
+  file?: string;
+};
+
+function readKnownNamesConfig(): unknown {
+  if (!existsSync(KNOWN_NAMES_PATH)) {
+    return null;
+  }
+
+  try {
+    return JSON.parse(readFileSync(KNOWN_NAMES_PATH, "utf-8")) as unknown;
+  } catch {
+    return null;
+  }
+}
+
+function isStringArray(value: unknown): value is string[] {
+  return Array.isArray(value) && value.every((item) => typeof item === "string");
+}
+
+function isStringRecord(value: unknown): value is Record<string, string> {
+  return (
+    typeof value === "object"
+    && value !== null
+    && !Array.isArray(value)
+    && Object.values(value).every((item) => typeof item === "string")
+  );
+}
+
+function _loadKnownPeople(): string[] {
+  const data = readKnownNamesConfig();
+  if (isStringArray(data)) {
+    return data;
+  }
+
+  if (typeof data === "object" && data !== null) {
+    const config = data as KnownNamesConfig;
+    if (isStringArray(config.names)) {
+      return config.names;
+    }
+  }
+
+  return FALLBACK_PEOPLE;
+}
+
+function _loadUsernameMap(): Record<string, string> {
+  const data = readKnownNamesConfig();
+  if (typeof data === "object" && data !== null && !Array.isArray(data)) {
+    const config = data as KnownNamesConfig;
+    if (isStringRecord(config.username_map)) {
+      return config.username_map;
+    }
+  }
+
+  return {};
+}
+
+const KNOWN_PEOPLE = _loadKnownPeople();
+
+function escapeRegex(text: string): string {
+  return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
+function sanitizeFileComponent(text: string): string {
+  return text.replace(/[^\w.-]/g, "_").replace(/_+/g, "_");
+}
+
+export function isTrueSessionStart(lines: string[], idx: number): boolean {
+  const nearby = lines.slice(idx, idx + 6).join("");
+  return !nearby.includes("Ctrl+E") && !nearby.includes("previous messages");
+}
+
+export function findSessionBoundaries(lines: string[]): number[] {
+  const boundaries: number[] = [];
+  for (const [index, line] of lines.entries()) {
+    if (line.includes("Claude Code v") && isTrueSessionStart(lines, index)) {
+      boundaries.push(index);
+    }
+  }
+  return boundaries;
+}
+
+export function extractTimestamp(lines: string[]): [string | null, string | null] {
+  for (const line of lines.slice(0, 50)) {
+    const match = TIMESTAMP_PATTERN.exec(line);
+    if (!match) {
+      continue;
+    }
+
+    const [, timeStr, month, day, year] = match;
+    const monthValue = MONTHS[month] ?? "00";
+    const dayValue = day.padStart(2, "0");
+    const timeSafe = timeStr.replace(":", "").replace(" ", "");
+    const iso = `${year}-${monthValue}-${dayValue}`;
+    const human = `${year}-${monthValue}-${dayValue}_${timeSafe}`;
+    return [human, iso];
+  }
+
+  return [null, null];
+}
+
+export function extractPeople(lines: string[]): string[] {
+  const found = new Set<string>();
+  const text = lines.slice(0, 100).join("");
+
+  for (const person of KNOWN_PEOPLE) {
+    const pattern = new RegExp(`\\b${escapeRegex(person)}\\b`, "i");
+    if (pattern.test(text)) {
+      found.add(person);
+    }
+  }
+
+  const dirMatch = /\/Users\/(\w+)\//.exec(text);
+  if (dirMatch) {
+    const usernameMap = _loadUsernameMap();
+    const username = dirMatch[1];
+    const mappedName = usernameMap[username];
+    if (mappedName) {
+      found.add(mappedName);
+    }
+  }
+
+  return Array.from(found).sort((a, b) => a.localeCompare(b));
+}
+
+export function extractSubject(lines: string[]): string {
+  for (const line of lines) {
+    if (!line.startsWith("> ")) {
+      continue;
+    }
+
+    const prompt = line.slice(2).trim();
+    if (!prompt || SUBJECT_SKIP_PATTERN.test(prompt) || prompt.length <= 5) {
+      continue;
+    }
+
+    const subject = prompt.replace(/[^\w\s-]/g, "").replace(/\s+/g, "-").trim();
+    return subject.slice(0, 60) || "session";
+  }
+
+  return "session";
+}
+
+export function splitFile(filepath: string, outputDir?: string, dryRun = false): string[] {
+  const content = readFileSync(filepath, "utf-8");
+  const lines = content.split(/(?<=\n)/u);
+  const boundaries = findSessionBoundaries(lines);
+
+  if (boundaries.length < 2) {
+    return [];
+  }
+
+  boundaries.push(lines.length);
+  const outDir = outputDir ?? parse(filepath).dir;
+  const written: string[] = [];
+
+  if (!dryRun) {
+    mkdirSync(outDir, { recursive: true });
+  }
+
+  for (let index = 0; index < boundaries.length - 1; index += 1) {
+    const start = boundaries[index];
+    const end = boundaries[index + 1];
+    const chunk = lines.slice(start, end);
+
+    if (chunk.length < 10) {
+      continue;
+    }
+
+    const [timestampHuman] = extractTimestamp(chunk);
+    const people = extractPeople(chunk);
+    const subject = extractSubject(chunk);
+    const timestampPart = timestampHuman ?? `part${String(index + 1).padStart(2, "0")}`;
+    const peoplePart = people.length > 0 ? people.slice(0, 3).join("-") : "unknown";
+    const sourceStem = parse(filepath).name.replace(/[^\w-]/g, "_").slice(0, 40);
+    const filename = sanitizeFileComponent(
+      `${sourceStem}__${timestampPart}_${peoplePart}_${subject}.txt`,
+    );
+    const outPath = join(outDir, filename);
+
+    if (dryRun) {
+      console.log(`  [${index + 1}/${boundaries.length - 1}] ${filename}  (${chunk.length} lines)`);
+    } else {
+      writeFileSync(outPath, chunk.join(""));
+      console.log(`  ✓ ${filename}  (${chunk.length} lines)`);
+    }
+
+    written.push(outPath);
+  }
+
+  return written;
+}
+
+export function splitMegaFiles(options: SplitMegaFilesOptions): string[] {
+  const srcDir = options.dir ?? LUMI_DIR;
+  const outputDir = options.outputDir;
+  const dryRun = options.dryRun ?? false;
+  const minSessions = options.minSessions ?? 2;
+  const files = options.file
+    ? [options.file]
+    : readdirSync(srcDir)
+      .filter((name) => name.endsWith(".txt"))
+      .sort((a, b) => a.localeCompare(b))
+      .map((name) => join(srcDir, name));
+
+  const megaFiles: Array<{ file: string; sessions: number }> = [];
+
+  for (const file of files) {
+    const lines = readFileSync(file, "utf-8").split(/(?<=\n)/u);
+    const boundaries = findSessionBoundaries(lines);
+    if (boundaries.length >= minSessions) {
+      megaFiles.push({ file, sessions: boundaries.length });
+    }
+  }
+
+  if (megaFiles.length === 0) {
+    console.log(`No mega-files found in ${srcDir} (min ${minSessions} sessions).`);
+    return [];
+  }
+
+  console.log(`\n${"=".repeat(60)}`);
+  console.log(`  Mega-file splitter — ${dryRun ? "DRY RUN" : "SPLITTING"}`);
+  console.log(`${"=".repeat(60)}`);
+  console.log(`  Source:      ${srcDir}`);
+  console.log(`  Output:      ${outputDir ?? "same dir as source"}`);
+  console.log(`  Mega-files:  ${megaFiles.length}`);
+  console.log(`${"─".repeat(60)}\n`);
+
+  const allWritten: string[] = [];
+
+  for (const megaFile of megaFiles) {
+    const sizeKb = Math.floor(statSync(megaFile.file).size / 1024);
+    console.log(`  ${basename(megaFile.file)}  (${megaFile.sessions} sessions, ${sizeKb}KB)`);
+    const written = splitFile(megaFile.file, outputDir, dryRun);
+    allWritten.push(...written);
+
+    if (!dryRun && written.length > 0) {
+      const backup = join(parse(megaFile.file).dir, `${parse(megaFile.file).name}.mega_backup`);
+      renameSync(megaFile.file, backup);
+      console.log(`  → Original renamed to ${basename(backup)}\n`);
+    } else {
+      console.log("");
+    }
+  }
+
+  console.log(`${"─".repeat(60)}`);
+  if (dryRun) {
+    console.log(`  DRY RUN — would create ${allWritten.length} files from ${megaFiles.length} mega-files`);
+  } else {
+    console.log(`  Done — created ${allWritten.length} files from ${megaFiles.length} mega-files`);
+  }
+  console.log("");
+
+  return allWritten;
+}