ex-brain 0.1.0 → 0.1.1

package/src/config.ts

@@ -0,0 +1,80 @@
+import { pinyin } from "pinyin-pro";
+
+export const DEFAULT_DB_NAME = "ebrain";
+export const PAGES_COLLECTION = "ebrain_pages";
+export const MAX_SLUG_LENGTH = 100;
+
+export function nowIso(): string {
+  return new Date().toISOString();
+}
+
+export function slugToTitle(slug: string): string {
+  const base = slug.split("/").at(-1) ?? slug;
+  return base
+    .split("-")
+    .filter(Boolean)
+    .map((part) => part[0]!.toUpperCase() + part.slice(1))
+    .join(" ");
+}
+
+export function inferTypeFromSlug(slug: string): string {
+  return slug.split("/")[0] ?? "other";
+}
+
+/**
+ * 将包含中文、空格、特殊字符的原始 slug 转换为
+ * 全英文小写 + 下划线连接的规范化格式。
+ * - 中文 → 拼音（无音调），英文单词保持完整不逐字母拆分
+ * - 空格/连字符/斜杠/点 → 下划线
+ * - 移除非字母数字字符
+ * - 截断至 MAX_SLUG_LENGTH
+ */
+export function slugify(input: string, maxLen = MAX_SLUG_LENGTH): string {
+  // 1. 分段处理：中文字符转拼音，非中文保持原样
+  let slug = "";
+  let chineseBuf = "";
+
+  function flushChinese() {
+    if (chineseBuf.length > 0) {
+      slug += " " + pinyin(chineseBuf, { toneType: "none", type: "string" });
+      chineseBuf = "";
+    }
+  }
+
+  for (const ch of input) {
+    // CJK Unified Ideographs range
+    if (ch >= "\u4e00" && ch <= "\u9fff") {
+      chineseBuf += ch;
+    } else {
+      flushChinese();
+      slug += ch;
+    }
+  }
+  flushChinese();
+
+  // 2. 转小写
+  slug = slug.toLowerCase();
+
+  // 3. 将所有非字母数字字符替换为下划线
+  slug = slug.replace(/[^a-z0-9]+/g, "_");
+
+  // 4. 合并连续下划线、去除首尾下划线
+  slug = slug.replace(/_+/g, "_").replace(/^_|_$/g, "");
+
+  // 5. 截断（确保不在下划线中间截断）
+  if (slug.length > maxLen) {
+    slug = slug.slice(0, maxLen).replace(/_+$/, "");
+  }
+
+  return slug || "untitled";
+}
+
+/**
+ * 仅当 slug 超过阈值时才进行规范化，避免改动已经合理的短 slug。
+ */
+export function normalizeLongSlug(slug: string): string {
+  if (slug.length > MAX_SLUG_LENGTH) {
+    return slugify(slug);
+  }
+  return slug;
+}

package/src/db/client.ts

@@ -0,0 +1,101 @@
+import { mkdir } from "node:fs/promises";
+import { dirname } from "node:path";
+import { SeekdbAdminClient, SeekdbClient, DEFAULT_PORT, DEFAULT_USER } from "seekdb";
+import type { Collection } from "seekdb";
+import type { ResolvedSettings } from "../settings";
+import { createBrainEmbeddingFunction } from "../ai/embed-factory";
+import { DEFAULT_DB_NAME, PAGES_COLLECTION } from "../config";
+import { SQL_SCHEMA } from "./schema";
+
+function useRemoteSeekdb(): boolean {
+  return Boolean(process.env.EBRAIN_SEEKDB_HOST?.trim());
+}
+
+function seekdbPassword(): string {
+  return process.env.EBRAIN_SEEKDB_PASSWORD ?? process.env.SEEKDB_PASSWORD ?? "";
+}
+
+export class BrainDb {
+  private constructor(
+    public readonly dbPath: string,
+    public readonly client: SeekdbClient,
+    public readonly pagesCollection: Collection,
+  ) {}
+
+  static async connect(dbPath: string, settings?: ResolvedSettings): Promise<BrainDb> {
+    const client = settings?.remote
+      ? await BrainDb.openRemoteClient(settings.remote)
+      : useRemoteSeekdb()
+        ? await BrainDb.openRemoteClientFromEnv()
+        : await BrainDb.openEmbeddedClient(dbPath);
+
+    for (const sql of SQL_SCHEMA) {
+      await client.execute(sql);
+    }
+
+    const pagesCollection = await client.getOrCreateCollection({
+      name: PAGES_COLLECTION,
+      embeddingFunction: createBrainEmbeddingFunction(settings?.embed),
+    });
+
+    return new BrainDb(dbPath, client, pagesCollection);
+  }
+
+  private static async openEmbeddedClient(dbPath: string): Promise<SeekdbClient> {
+    await mkdir(dirname(dbPath), { recursive: true });
+    const admin = new SeekdbAdminClient({ path: dbPath });
+    try {
+      await admin.createDatabase(DEFAULT_DB_NAME);
+      await admin.getDatabase(DEFAULT_DB_NAME);
+    } catch (error) {
+      try {
+        await admin.getDatabase(DEFAULT_DB_NAME);
+      } catch {
+        throw error;
+      }
+    }
+
+    return new SeekdbClient({
+      path: dbPath,
+      database: DEFAULT_DB_NAME,
+    });
+  }
+
+  private static async openRemoteClient(remote: NonNullable<ResolvedSettings["remote"]>): Promise<SeekdbClient> {
+    const args: ConstructorParameters<typeof SeekdbClient>[0] = {
+      host: remote.host,
+      port: remote.port,
+      user: remote.user,
+      password: remote.password,
+      database: remote.database,
+    };
+    if (remote.tenant) {
+      args.tenant = remote.tenant;
+    }
+    return new SeekdbClient(args);
+  }
+
+  private static async openRemoteClientFromEnv(): Promise<SeekdbClient> {
+    const host = process.env.EBRAIN_SEEKDB_HOST!.trim();
+    const port = Number(process.env.EBRAIN_SEEKDB_PORT ?? DEFAULT_PORT);
+    const user = process.env.EBRAIN_SEEKDB_USER ?? DEFAULT_USER;
+    const database =
+      process.env.EBRAIN_SEEKDB_DATABASE?.trim() || DEFAULT_DB_NAME;
+    const tenant = process.env.EBRAIN_SEEKDB_TENANT?.trim();
+    const args: ConstructorParameters<typeof SeekdbClient>[0] = {
+      host,
+      port,
+      user,
+      password: seekdbPassword(),
+      database,
+    };
+    if (tenant) {
+      args.tenant = tenant;
+    }
+    return new SeekdbClient(args);
+  }
+
+  async close(): Promise<void> {
+    await this.client.close();
+  }
+}

package/src/db/schema.ts

@@ -0,0 +1,49 @@
+/** DDL compatible with seekdb embedded (MySQL-style). TEXT/MEDIUMTEXT cannot have DEFAULT in strict engines. */
+export const SQL_SCHEMA = [
+  `CREATE TABLE IF NOT EXISTS pages (
+    slug VARCHAR(768) PRIMARY KEY,
+    type VARCHAR(128) NOT NULL,
+    title VARCHAR(512) NOT NULL,
+    compiled_truth MEDIUMTEXT NOT NULL,
+    timeline MEDIUMTEXT NOT NULL,
+    frontmatter MEDIUMTEXT NOT NULL,
+    created_at VARCHAR(64) NOT NULL,
+    updated_at VARCHAR(64) NOT NULL
+  )`,
+  `CREATE TABLE IF NOT EXISTS links (
+    from_slug VARCHAR(768) NOT NULL,
+    to_slug VARCHAR(768) NOT NULL,
+    context MEDIUMTEXT NOT NULL,
+    created_at VARCHAR(64) NOT NULL,
+    UNIQUE(from_slug, to_slug)
+  )`,
+  `CREATE TABLE IF NOT EXISTS timeline_entries (
+    id BIGINT AUTO_INCREMENT PRIMARY KEY,
+    page_slug VARCHAR(768) NOT NULL,
+    date VARCHAR(32) NOT NULL,
+    source VARCHAR(128) NOT NULL,
+    summary VARCHAR(1024) NOT NULL,
+    detail MEDIUMTEXT NOT NULL,
+    created_at VARCHAR(64) NOT NULL
+  )`,
+  `CREATE TABLE IF NOT EXISTS raw_data (
+    id BIGINT AUTO_INCREMENT PRIMARY KEY,
+    page_slug VARCHAR(768) NOT NULL,
+    source VARCHAR(256) NOT NULL,
+    data MEDIUMTEXT NOT NULL,
+    fetched_at VARCHAR(64) NOT NULL
+  )`,
+  `CREATE TABLE IF NOT EXISTS page_tags (
+    page_slug VARCHAR(768) NOT NULL,
+    tag VARCHAR(256) NOT NULL,
+    created_at VARCHAR(64) NOT NULL,
+    UNIQUE(page_slug, tag)
+  )`,
+  `CREATE TABLE IF NOT EXISTS ingest_log (
+    id BIGINT AUTO_INCREMENT PRIMARY KEY,
+    source_ref VARCHAR(1024) NOT NULL,
+    source_type VARCHAR(128) NOT NULL,
+    detail MEDIUMTEXT NOT NULL,
+    created_at VARCHAR(64) NOT NULL
+  )`,
+];

package/src/markdown/io.ts

@@ -0,0 +1,61 @@
+import { mkdir, readdir, readFile, stat, writeFile } from "node:fs/promises";
+import { dirname, extname, join, relative, resolve } from "node:path";
+
+export async function readMaybeStdin(): Promise<string | null> {
+  if (process.stdin.isTTY) return null;
+  const chunks: Buffer[] = [];
+  for await (const chunk of process.stdin) {
+    chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
+  }
+  const text = Buffer.concat(chunks).toString("utf8");
+  return text;
+}
+
+export async function collectMarkdownFiles(dir: string): Promise<string[]> {
+  const root = resolve(dir);
+  const files: string[] = [];
+  async function walk(current: string): Promise<void> {
+    const entries = await readdir(current, { withFileTypes: true });
+    for (const entry of entries) {
+      const next = join(current, entry.name);
+      if (entry.isDirectory()) {
+        await walk(next);
+      } else if (entry.isFile() && extname(entry.name) === ".md") {
+        files.push(next);
+      }
+    }
+  }
+  await walk(root);
+  return files.sort();
+}
+
+export function pathToSlug(filePath: string, rootDir: string): string {
+  const rel = relative(resolve(rootDir), resolve(filePath));
+  return rel.replace(/\.md$/, "").replaceAll("\\", "/");
+}
+
+export function slugToPath(slug: string, rootDir: string): string {
+  return join(resolve(rootDir), `${slug}.md`);
+}
+
+export async function writeTextFile(path: string, content: string): Promise<void> {
+  await mkdir(dirname(path), { recursive: true });
+  await writeFile(path, content, "utf8");
+}
+
+export async function readTextFile(path: string): Promise<string> {
+  return await readFile(path, "utf8");
+}
+
+export async function ensureDir(path: string): Promise<void> {
+  await mkdir(path, { recursive: true });
+}
+
+export async function fileExists(path: string): Promise<boolean> {
+  try {
+    const s = await stat(path);
+    return s.isFile() || s.isDirectory();
+  } catch {
+    return false;
+  }
+}

package/src/markdown/parser.ts

@@ -0,0 +1,72 @@
+import matter from "gray-matter";
+
+export interface ParsedMarkdownPage {
+  frontmatter: Record<string, unknown>;
+  compiledTruth: string;
+  timeline: string;
+}
+
+const SPLIT_MARKER = /^-{3,}\s*$/m;
+
+export function parsePageMarkdown(input: string): ParsedMarkdownPage {
+  const parsed = matter(input);
+  const content = parsed.content.trim();
+  const [compiledTruth, timeline] = splitCompiledAndTimeline(content);
+  return {
+    frontmatter: (parsed.data ?? {}) as Record<string, unknown>,
+    compiledTruth: compiledTruth.trim(),
+    timeline: timeline.trim(),
+  };
+}
+
+export function renderPageMarkdown(
+  frontmatter: Record<string, unknown>,
+  compiledTruth: string,
+  timeline: string,
+): string {
+  const body = [compiledTruth.trim(), "---", timeline.trim()]
+    .filter((part) => part.length > 0)
+    .join("\n\n");
+  return matter.stringify(body, frontmatter);
+}
+
+function splitCompiledAndTimeline(content: string): [string, string] {
+  const match = SPLIT_MARKER.exec(content);
+  if (!match || match.index === undefined) {
+    return [content, ""];
+  }
+  const left = content.slice(0, match.index);
+  const right = content.slice(match.index + match[0].length);
+  return [left, right];
+}
+
+export function extractWikiStyleLinks(content: string): string[] {
+  const regex = /\[[^\]]+\]\(([^)]+\.md)\)/g;
+  const links: string[] = [];
+  let m: RegExpExecArray | null;
+  while ((m = regex.exec(content)) !== null) {
+    if (m[1]) {
+      links.push(m[1]);
+    }
+  }
+  return links;
+}
+
+export function extractTimelineLines(
+  timelineMarkdown: string,
+): Array<{ date: string; source: string; summary: string }> {
+  const lines = timelineMarkdown.split("\n");
+  const results: Array<{ date: string; source: string; summary: string }> = [];
+  const re = /^\s*-\s+\*\*(\d{4}-\d{2}-\d{2})\*\*\s*\|\s*([^—-]+)[—-]\s*(.+)$/;
+  for (const line of lines) {
+    const m = line.match(re);
+    if (m) {
+      results.push({
+        date: m[1],
+        source: m[2].trim(),
+        summary: m[3].trim(),
+      });
+    }
+  }
+  return results;
+}