ex-brain 0.2.7 → 0.3.0

package/src/commands/timeline-cmd.ts

@@ -0,0 +1,159 @@
+import { Command } from "commander";
+import { loadSettings } from "../settings";
+import { addDryRun, isDryRun, withRepo, isJson, print } from "./shared";
+import { createProgress, formatDuration } from "../utils/progress";
+
+export function registerTimelineCommand(program: Command): void {
+  const timelineCmd = program
+    .command("timeline")
+    .description("manage timeline entries");
+
+  // timeline list
+  timelineCmd
+    .command("list")
+    .argument("<slug>", "page slug")
+    .option("--limit <number>", "max results", "50")
+    .description("list timeline entries for a page")
+    .addHelpText("after", `
+Examples:
+  ebrain timeline list projects/alpha
+  ebrain timeline list projects/alpha --limit 10
+`)
+    .action(async (slug: string, opts: Record<string, string>) => {
+      await withRepo(program, async (repo) => {
+        const rows = await repo.timeline(slug, Number(opts.limit ?? 50));
+        print(program, rows);
+      });
+    });
+
+  // timeline add
+  addDryRun(
+    timelineCmd
+      .command("add")
+      .argument("<slug>", "page slug")
+      .requiredOption("--date <date>", "date (YYYY-MM-DD or ISO)")
+      .requiredOption("--summary <summary>", "one-line summary")
+      .option("--source <source>", "event source", "manual")
+      .option("--detail <detail>", "detail markdown", "")
+      .description("add a timeline entry")
+      .addHelpText("after", `
+Examples:
+  ebrain timeline add projects/alpha --date 2025-03-15 --summary "v1.0 shipped"
+  ebrain timeline add projects/alpha --date 2025-03-15 --summary "launch" --source release
+  ebrain timeline add projects/alpha --date 2025-03-15 --summary "launch" --dry-run
+`),
+  ).action(async (slug: string, opts: {
+    date: string;
+    summary: string;
+    source?: string;
+    detail?: string;
+    dryRun?: boolean;
+  }) => {
+    if (isDryRun(opts)) {
+      print(program, {
+        dryRun: true,
+        action: "timeline-add",
+        slug,
+        date: opts.date,
+        summary: opts.summary,
+        source: opts.source ?? "manual",
+      });
+      return;
+    }
+    await withRepo(program, async (repo) => {
+      await repo.timelineAdd({
+        pageSlug: slug,
+        date: opts.date,
+        source: opts.source ?? "manual",
+        summary: opts.summary,
+        detail: opts.detail ?? "",
+      });
+      print(program, {
+        ok: true,
+        action: "timeline-add",
+        slug,
+        date: opts.date,
+      });
+    });
+  });
+
+  // timeline extract
+  addDryRun(
+    timelineCmd
+      .command("extract")
+      .argument("<slug>", "page slug")
+      .option("--source <source>", "source identifier", "extracted")
+      .option("--default-date <date>", "default date (YYYY-MM-DD)")
+      .description("extract timeline events from page content using AI")
+      .addHelpText("after", `
+Examples:
+  ebrain timeline extract companies/river-ai
+  ebrain timeline extract docs/meeting --source meeting_notes --default-date 2024-03-15
+`),
+  ).action(async (slug: string, opts: { source?: string; defaultDate?: string; dryRun?: boolean }) => {
+    if (isDryRun(opts)) {
+      print(program, {
+        dryRun: true,
+        action: "timeline-extract",
+        slug,
+        source: opts.source ?? "extracted",
+        defaultDate: opts.defaultDate ?? new Date().toISOString().slice(0, 10),
+      });
+      return;
+    }
+    await withRepo(program, async (repo) => {
+      const page = await repo.getPage(slug);
+      if (!page) {
+        throw new Error(`page not found: ${slug}`);
+      }
+      const settings = await loadSettings();
+
+      const progress = createProgress();
+      progress.start(`Extracting timeline from ${slug}...`);
+      const startTime = Date.now();
+
+      const result = await repo.extractAndAddTimeline(
+        slug,
+        page.compiledTruth,
+        opts.source ?? "extracted",
+        opts.defaultDate ?? new Date().toISOString().slice(0, 10),
+        settings.llm,
+      );
+
+      const duration = formatDuration(Date.now() - startTime);
+
+      if (result.entries.length > 0) {
+        progress.succeed(`${result.entries.length} events extracted (${duration})`);
+      } else {
+        progress.stop();
+        process.stderr.write(`No events found (${duration})\n`);
+      }
+
+      print(program, {
+        ok: true,
+        action: "timeline-extract",
+        slug,
+        entriesAdded: result.entries.length,
+        entries: result.entries,
+        confidence: result.confidence,
+      });
+    });
+  });
+
+  // timeline global
+  timelineCmd
+    .command("global")
+    .option("--limit <number>", "max results", "100")
+    .description("list timeline entries across all pages")
+    .addHelpText("after", `
+Examples:
+  ebrain timeline global
+  ebrain timeline global --limit 20
+`)
+    .action(async (opts: Record<string, string>) => {
+      await withRepo(program, async (repo) => {
+        const entries = await repo.timelineGlobal(Number(opts.limit ?? 100));
+        print(program, entries);
+      });
+    });
+}

package/src/config/index.ts

@@ -0,0 +1,53 @@
+// ---------------------------------------------------------------------------
+// Public API — re-export from individual modules
+// ---------------------------------------------------------------------------
+
+export {
+  SETTINGS_DIR,
+  SETTINGS_PATH,
+  DEFAULT_DB_PATH,
+  expandTilde,
+} from "./paths";
+
+export {
+  SettingsSchema,
+  RemoteDbSchema,
+  EmbedSchema,
+  LLMSchema,
+  type RawSettings,
+  type ResolvedSettings,
+  type ResolvedExtraction,
+  type ResolvedRemoteDb,
+  type ResolvedEmbed,
+  type ResolvedLLM,
+  DEFAULT_REMOTE,
+  DEFAULT_EMBED,
+  DEFAULT_LLM,
+  DEFAULT_EXTRACTION,
+} from "./schema";
+
+export {
+  type EnvSource,
+  readSettingsFile,
+  resolveSettings,
+  loadSettings,
+} from "./settings";
+
+export {
+  createDefaultSettings,
+} from "./init";
+
+// ---------------------------------------------------------------------------
+// Slug utilities (from sibling file)
+// ---------------------------------------------------------------------------
+
+export {
+  DEFAULT_DB_NAME,
+  PAGES_COLLECTION,
+  MAX_SLUG_LENGTH,
+  nowIso,
+  slugToTitle,
+  inferTypeFromSlug,
+  slugify,
+  normalizeLongSlug,
+} from "../slug-utils";

package/src/config/init.ts

@@ -0,0 +1,50 @@
+import { SETTINGS_PATH, SETTINGS_DIR } from "./paths";
+import { fileExists } from "../markdown/io";
+import { mkdirSync, writeFileSync } from "node:fs";
+
+/**
+ * Generate a minimal settings.json if it doesn't already exist.
+ * Returns true if a new file was created.
+ */
+export async function createDefaultSettings(): Promise<boolean> {
+  if (await fileExists(SETTINGS_PATH)) {
+    return false;
+  }
+
+  mkdirSync(SETTINGS_DIR, { recursive: true });
+
+  // All fields present but empty — user fills in their values
+  const defaults = {
+    db: {
+      path: "",
+      remote: {
+        host: "",
+        port: 0,
+        user: "",
+        password: "",
+        database: "",
+        tenant: "",
+      },
+    },
+    embed: {
+      provider: "hash",
+      baseURL: "",
+      model: "",
+      dimensions: 0,
+      apiKey: "",
+      apiKeyEnv: "",
+    },
+    llm: {
+      baseURL: "",
+      model: "",
+      apiKey: "",
+      apiKeyEnv: "",
+    },
+    extraction: {
+      confidenceThreshold: 0.7,
+    },
+  };
+
+  writeFileSync(SETTINGS_PATH, JSON.stringify(defaults, null, 2) + "\n", "utf-8");
+  return true;
+}

package/src/config/paths.ts

@@ -0,0 +1,21 @@
+import { homedir } from "node:os";
+import { join, resolve } from "node:path";
+
+/** Settings directory: ~/.ebrain */
+export const SETTINGS_DIR = join(homedir(), ".ebrain");
+
+/** Settings file path: ~/.ebrain/settings.json */
+export const SETTINGS_PATH = join(SETTINGS_DIR, "settings.json");
+
+/** Default database path: ~/.ebrain/data/ebrain.db */
+export const DEFAULT_DB_PATH = resolve(SETTINGS_DIR, "data", "ebrain.db");
+
+/**
+ * Resolve a path that may start with ~ to the user's home directory.
+ */
+export function expandTilde(p: string): string {
+  if (p.startsWith("~")) {
+    return join(homedir(), p.slice(1));
+  }
+  return resolve(p);
+}

package/src/config/schema.ts

@@ -0,0 +1,121 @@
+import { z } from "zod";
+
+// ---------------------------------------------------------------------------
+// Raw schema (matches settings.json structure)
+// ---------------------------------------------------------------------------
+
+export const RemoteDbSchema = z.object({
+  host: z.string().optional(),
+  port: z.number().optional(),
+  user: z.string().optional(),
+  password: z.string().optional(),
+  database: z.string().optional(),
+  tenant: z.string().optional(),
+});
+
+export const EmbedSchema = z.object({
+  provider: z.enum(["hash", "openai_compatible"]).optional(),
+  baseURL: z.string().optional(),
+  model: z.string().optional(),
+  dimensions: z.number().optional(),
+  apiKey: z.string().optional(),
+  apiKeyEnv: z.string().optional(),
+});
+
+export const LLMSchema = z.object({
+  baseURL: z.string().optional(),
+  model: z.string().optional(),
+  apiKey: z.string().optional(),
+  apiKeyEnv: z.string().optional(),
+});
+
+export const SettingsSchema = z.object({
+  db: z
+    .object({
+      path: z.string().optional(),
+      remote: RemoteDbSchema.optional(),
+    })
+    .optional(),
+  embed: EmbedSchema.optional(),
+  llm: LLMSchema.optional(),
+  extraction: z
+    .object({
+      confidenceThreshold: z.number().min(0).max(1).optional(),
+    })
+    .optional(),
+});
+
+export type RawSettings = z.infer<typeof SettingsSchema>;
+
+// ---------------------------------------------------------------------------
+// Resolved types (all values present after defaults + env merge)
+// ---------------------------------------------------------------------------
+
+export interface ResolvedSettings {
+  dbPath: string;
+  remote: ResolvedRemoteDb | null;
+  embed: ResolvedEmbed;
+  llm: ResolvedLLM;
+  extraction: ResolvedExtraction;
+}
+
+export interface ResolvedExtraction {
+  confidenceThreshold: number;
+}
+
+export interface ResolvedRemoteDb {
+  host: string;
+  port: number;
+  user: string;
+  password: string;
+  database: string;
+  tenant: string;
+}
+
+export interface ResolvedEmbed {
+  provider: "hash" | "openai_compatible";
+  baseURL: string;
+  model: string;
+  dimensions: number;
+  apiKey: string;
+  apiKeyEnv: string;
+}
+
+export interface ResolvedLLM {
+  baseURL: string;
+  model: string;
+  apiKey: string;
+  apiKeyEnv: string;
+}
+
+// ---------------------------------------------------------------------------
+// Default values
+// ---------------------------------------------------------------------------
+
+export const DEFAULT_REMOTE: Omit<ResolvedRemoteDb, "host"> = {
+  port: 3306,
+  user: "root",
+  password: "",
+  database: "ebrain",
+  tenant: "",
+};
+
+export const DEFAULT_EMBED: Omit<ResolvedEmbed, "provider"> & { provider: "hash" } = {
+  provider: "hash",
+  baseURL: "https://dashscope.aliyuncs.com/compatible-mode/v1",
+  model: "text-embedding-v4",
+  dimensions: 1024,
+  apiKey: "",
+  apiKeyEnv: "DASHSCOPE_API_KEY",
+};
+
+export const DEFAULT_LLM: ResolvedLLM = {
+  baseURL: "",
+  model: "qwen-plus",
+  apiKey: "",
+  apiKeyEnv: "DASHSCOPE_API_KEY",
+};
+
+export const DEFAULT_EXTRACTION: ResolvedExtraction = {
+  confidenceThreshold: 0.7,
+};

package/src/config/settings.ts

@@ -0,0 +1,168 @@
+import { SETTINGS_PATH, SETTINGS_DIR, expandTilde, DEFAULT_DB_PATH } from "./paths";
+import {
+  SettingsSchema,
+  DEFAULT_REMOTE,
+  DEFAULT_EMBED,
+  DEFAULT_LLM,
+  DEFAULT_EXTRACTION,
+  type RawSettings,
+  type ResolvedSettings,
+  type ResolvedRemoteDb,
+  type ResolvedEmbed,
+  type ResolvedLLM,
+  type ResolvedExtraction,
+} from "./schema";
+
+// ---------------------------------------------------------------------------
+// Env abstraction for testability
+// ---------------------------------------------------------------------------
+
+export interface EnvSource {
+  get(key: string): string | undefined;
+}
+
+const defaultEnv: EnvSource = { get: (k) => process.env[k] };
+
+// ---------------------------------------------------------------------------
+// File I/O
+// ---------------------------------------------------------------------------
+
+export async function readSettingsFile(): Promise<unknown | null> {
+  const { fileExists, readTextFile } = await import("../markdown/io");
+  if (!(await fileExists(SETTINGS_PATH))) {
+    return null;
+  }
+  const text = await readTextFile(SETTINGS_PATH);
+  try {
+    return JSON.parse(text) as unknown;
+  } catch {
+    console.warn(
+      `[ebrain] Failed to parse ${SETTINGS_PATH}, using defaults.`,
+    );
+    return null;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Resolution: raw settings + env → resolved settings
+// ---------------------------------------------------------------------------
+
+export function resolveSettings(
+  parsed: RawSettings,
+  env: EnvSource = defaultEnv,
+): ResolvedSettings {
+  const dbConf = parsed.db ?? {};
+  const remoteConf = dbConf.remote ?? {};
+  const embedConf = parsed.embed ?? {};
+  const extractionConf = parsed.extraction ?? {};
+
+  // Remote: settings → env → defaults
+  const host = nonEmpty(remoteConf.host ?? env.get("EBRAIN_SEEKDB_HOST"), "");
+  if (host) {
+    const remote: ResolvedRemoteDb = {
+      host: host.trim(),
+      port: numOr(remoteConf.port ?? env.get("EBRAIN_SEEKDB_PORT"), DEFAULT_REMOTE.port),
+      user: nonEmpty(remoteConf.user ?? env.get("EBRAIN_SEEKDB_USER"), DEFAULT_REMOTE.user),
+      password: nonEmpty(
+        remoteConf.password ?? env.get("EBRAIN_SEEKDB_PASSWORD"),
+        DEFAULT_REMOTE.password,
+      ),
+      database: nonEmpty(
+        remoteConf.database ?? env.get("EBRAIN_SEEKDB_DATABASE"),
+        DEFAULT_REMOTE.database,
+      ),
+      tenant: nonEmpty(remoteConf.tenant ?? env.get("EBRAIN_SEEKDB_TENANT"), ""),
+    };
+    return {
+      dbPath: dbConf.path ?? DEFAULT_DB_PATH,
+      remote,
+      embed: resolveEmbed(embedConf, env),
+      llm: resolveLLM(parsed.llm ?? {}, env),
+      extraction: resolveExtraction(extractionConf, env),
+    };
+  }
+
+  // Local mode
+  const dbPath = dbConf.path ? expandTilde(dbConf.path) : DEFAULT_DB_PATH;
+  return {
+    dbPath,
+    remote: null,
+    embed: resolveEmbed(embedConf, env),
+    llm: resolveLLM(parsed.llm ?? {}, env),
+    extraction: resolveExtraction(extractionConf, env),
+  };
+}
+
+function resolveEmbed(
+  conf: NonNullable<RawSettings["embed"]>,
+  env: EnvSource = defaultEnv,
+): ResolvedEmbed {
+  const provider = nonEmpty(
+    conf.provider ?? env.get("EBRAIN_EMBED_PROVIDER"),
+    DEFAULT_EMBED.provider,
+  ).trim().toLowerCase() as "hash" | "openai_compatible";
+  const baseURL = nonEmpty(conf.baseURL ?? env.get("EBRAIN_EMBED_BASE_URL"), DEFAULT_EMBED.baseURL);
+  const model = nonEmpty(conf.model ?? env.get("EBRAIN_EMBED_MODEL"), DEFAULT_EMBED.model);
+  const dimensions = numOr(conf.dimensions ?? env.get("EBRAIN_EMBED_DIMENSIONS"), DEFAULT_EMBED.dimensions);
+  const apiKey = nonEmpty(conf.apiKey ?? env.get("EBRAIN_EMBED_API_KEY"), "");
+  const apiKeyEnv = nonEmpty(conf.apiKeyEnv ?? env.get("EBRAIN_EMBED_API_KEY_ENV"), DEFAULT_EMBED.apiKeyEnv);
+  return { provider, baseURL, model, dimensions, apiKey, apiKeyEnv };
+}
+
+function resolveLLM(
+  conf: NonNullable<RawSettings["llm"]>,
+  env: EnvSource = defaultEnv,
+): ResolvedLLM {
+  const baseURL = nonEmpty(conf.baseURL, DEFAULT_LLM.baseURL);
+  const model = nonEmpty(conf.model, DEFAULT_LLM.model);
+  const apiKey = nonEmpty(conf.apiKey, DEFAULT_LLM.apiKey);
+  const apiKeyEnv = nonEmpty(conf.apiKeyEnv, DEFAULT_LLM.apiKeyEnv);
+  return { baseURL, model, apiKey, apiKeyEnv };
+}
+
+function resolveExtraction(
+  conf: NonNullable<RawSettings["extraction"]>,
+  env: EnvSource = defaultEnv,
+): ResolvedExtraction {
+  const threshold = conf.confidenceThreshold ?? env.get("EBRAIN_CONFIDENCE_THRESHOLD");
+  const value = typeof threshold === "number"
+    ? threshold
+    : (threshold ? parseFloat(threshold) : DEFAULT_EXTRACTION.confidenceThreshold);
+  return { confidenceThreshold: Math.max(0, Math.min(1, value)) };
+}
+
+// ---------------------------------------------------------------------------
+// Public load function
+// ---------------------------------------------------------------------------
+
+export async function loadSettings(
+  env: EnvSource = defaultEnv,
+): Promise<ResolvedSettings> {
+  const raw = await readSettingsFile();
+  const parsed = SettingsSchema.parse(raw ?? {});
+  return resolveSettings(parsed, env);
+}
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function nonEmpty(val: string | undefined, fallback: string): string {
+  const trimmed = val?.trim();
+  return trimmed || fallback;
+}
+
+function numOr(val: number | string | undefined, fallback: number): number {
+  if (typeof val === "number") return val;
+  if (typeof val === "string") {
+    const n = Number(val.trim());
+    if (Number.isFinite(n)) return n;
+  }
+  return fallback;
+}
+
+// ---------------------------------------------------------------------------
+// Re-export paths for backward compatibility
+// ---------------------------------------------------------------------------
+
+export { SETTINGS_DIR, SETTINGS_PATH, DEFAULT_DB_PATH, expandTilde };

package/src/db/client.ts

@@ -4,7 +4,7 @@ import { SeekdbAdminClient, SeekdbClient, DEFAULT_PORT, DEFAULT_USER } from "see
 import type { Collection } from "seekdb";
 import type { ResolvedSettings } from "../settings";
 import { createBrainEmbeddingFunction } from "../ai/embed-factory";
-import { DEFAULT_DB_NAME, PAGES_COLLECTION } from "../config";
+import { DEFAULT_DB_NAME, PAGES_COLLECTION } from "../slug-utils";
 import { SQL_SCHEMA } from "./schema";
 import { DbError, wrapDbError, DbErrorCategory } from "./errors";
 

package/src/markdown/document-loader.ts

@@ -1,5 +1,5 @@
-import { readFile, stat } from "node:fs/promises";
-import { basename, extname, resolve } from "node:path";
+import { readFile, readdir, stat } from "node:fs/promises";
+import { basename, extname, join, resolve } from "node:path";
 
 /** Supported document kinds for ingestion. */
 export type DocumentKind =
@@ -484,3 +484,31 @@ function looksLikeText(bytes: Buffer): boolean {
   }
   return textLike / sample.length >= 0.95;
 }
+
+/** File extensions eligible for document ingestion (binary/office formats). */
+const DOCUMENT_EXTENSIONS = new Set(["pdf", "docx"]);
+
+/**
+ * Recursively collect `.docx` and `.pdf` files under `dir`.
+ * Returns sorted absolute paths.
+ */
+export async function collectDocumentFiles(dir: string): Promise<string[]> {
+  const root = resolve(dir);
+  const files: string[] = [];
+  async function walk(current: string): Promise<void> {
+    const entries = await readdir(current, { withFileTypes: true });
+    for (const entry of entries) {
+      const next = join(current, entry.name);
+      if (entry.isDirectory()) {
+        await walk(next);
+      } else if (
+        entry.isFile() &&
+        DOCUMENT_EXTENSIONS.has(extname(entry.name).toLowerCase().replace(/^\./, ""))
+      ) {
+        files.push(next);
+      }
+    }
+  }
+  await walk(root);
+  return files.sort();
+}

package/src/repositories/brain-repo.ts

@@ -1,4 +1,4 @@
-import { nowIso } from "../config";
+import { nowIso } from "../slug-utils";
 import type {
   BrainStats,
   PageRecord,
@@ -630,6 +630,48 @@ export class BrainRepository {
     }
   }
 
+  /**
+   * Sync tags from frontmatter to the page_tags table.
+   * This ensures `ebrain list --tag <tag>` works correctly for pages
+   * created via `put` with frontmatter tags.
+   */
+  async syncTagsFromFrontmatter(slug: string, frontmatter: Record<string, unknown>): Promise<number> {
+    const fmTags = frontmatter.tags;
+    if (!fmTags) return 0;
+
+    const tags: string[] = Array.isArray(fmTags)
+      ? fmTags.filter((t): t is string => typeof t === "string")
+      : typeof fmTags === "string"
+        ? [fmTags]
+        : [];
+
+    if (tags.length === 0) return 0;
+
+    // Get current DB tags for this page
+    const existingTags = new Set(await this.tags(slug));
+    const desiredTags = new Set(tags);
+
+    let synced = 0;
+
+    // Add missing tags
+    for (const tag of desiredTags) {
+      if (!existingTags.has(tag)) {
+        await this.tag(slug, tag);
+        synced++;
+      }
+    }
+
+    // Remove tags no longer in frontmatter
+    for (const tag of existingTags) {
+      if (!desiredTags.has(tag)) {
+        await this.untag(slug, tag);
+        synced++;
+      }
+    }
+
+    return synced;
+  }
+
   async readRaw(slug: string, source?: string): Promise<unknown[]> {
     try {
       const params: unknown[] = [slug];